Skip to content
Snippets Groups Projects
Unverified Commit 6e35e0f1 authored by Aluísio Augusto Silva Gonçalves's avatar Aluísio Augusto Silva Gonçalves
Browse files

Mark resources as processed

parent 645ce65d
No related branches found
No related tags found
No related merge requests found
......@@ -24,4 +24,8 @@ interface Resource {
/** Original address where this resource was made available. */
var url: URL
/** Whether references have already been extracted from this
* resource or not. */
var processed: Boolean
}
......@@ -38,13 +38,11 @@ class JpaDao(
}
override fun getUnprocessedResources(): Flowable<out Resource> {
// TODO: mark processed resources as such
return getAllResources()
return resourceRepository.findByProcessedFalse().flow()
}
override fun countUnprocessedResources(): Long {
// TODO: mark processed resources as such
return countAllResources()
return resourceRepository.countByProcessedFalse()
}
override fun getAllReferences(): Flowable<out Reference> {
......
......@@ -26,7 +26,10 @@ data class JpaResource(
private var work: JpaWork,
@Column(name = "url", length = 4096)
override var url: URL
override var url: URL,
@Column(name = "processed")
override var processed: Boolean = false
) : Resource {
override val key: String
get() = id?.toString(10)
......@@ -64,4 +67,7 @@ data class JpaResource(
@Repository
interface ResourceRepository : CrudRepository<JpaResource, Long> {
fun findByWorkAndUrl(embodiedWork: JpaWork, url: URL): JpaResource?
fun findByProcessedFalse(): Iterable<JpaResource>
fun countByProcessedFalse(): Long
}
......@@ -23,7 +23,10 @@ data class SdnResource(
@Property(name = "url")
@Convert(UrlConverter::class)
override var url: URL
override var url: URL,
@Property(name = "processed")
override var processed: Boolean = false
) : Resource {
override val key: String
get() = nodeId?.toString(10)
......@@ -58,5 +61,8 @@ interface ResourceRepository : GraphRepository<SdnResource> {
@Query("MATCH (res:Manifestation) WHERE res.cacheId IS NULL RETURN ref")
fun getNotCached(): Iterable<SdnResource>
fun findByProcessedFalse(): Iterable<SdnResource>
fun countByProcessedFalse(): Long
}
......@@ -38,13 +38,11 @@ class SdnDao(
}
override fun getUnprocessedResources(): Flowable<out Resource> {
// TODO: mark processed resources as such
return getAllResources()
return resourceRepository.findByProcessedFalse().flow()
}
override fun countUnprocessedResources(): Long {
// TODO: mark processed resources as such
return countAllResources()
return resourceRepository.countByProcessedFalse()
}
override fun getAllReferences(): Flowable<out Reference> {
......
......@@ -27,7 +27,10 @@ class ReferenceExtractor(
log.info("Processing {}", res)
try {
resourceReader.read(res).use { pdf ->
extractReferences(res, pdf)
val references = extractReferences(res, pdf)
res.processed = true
dao.saveResource(res)
references
}
} catch (ex: Exception) {
log.catching(ex)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment