saveweb/cnblogs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 4 Wiki Activity

feat: handle infinite loops gracefully
Some checks failed
Gitea Go Release Actions / Release Go Binary (amd64, linux) (push) Has been cancelled
Details
Gitea Go Release Actions / Release Go Binary (arm64, darwin) (push) Has been cancelled
Details
Gitea Go Release Actions / Release Go Binary (arm64, linux) (push) Has been cancelled
Details
Gitea Go Release Actions / Release Go Binary (arm, linux) (push) Has been cancelled
Details
Gitea Go Release Actions / Release Go Binary (amd64, windows) (push) Has been cancelled
Details
Gitea Go Release Actions / Release Go Binary (amd64, darwin) (push) Has been cancelled
Details

Browse Source
This commit is contained in:
yzqzss 2024-07-18 16:59:30 +08:00
parent 61816823df
commit 2d75209b43
1 changed files with 34 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
cmd/cnblogs_posts_list/cnblogs_posts_list.go
View File

@ -69,6 +69,30 @@ func claimWorker(i int, tracker *savewebtracker.Tracker) {
}
}
// Refactored deduplication function
func dedup_append(allPostMetas, newPostMetas []cnblogs_api.PostMeta) ([]cnblogs_api.PostMeta, int) {
founds := 0
for _, newPostMeta := range newPostMetas {
_found := false
for all_idx, allPostMeta := range allPostMetas {
if newPostMeta.URL == allPostMeta.URL {
_found = true
founds++
// replace the old
Logger.Println("dedup_append: replace the old", allPostMeta, "with", newPostMeta)
allPostMetas[all_idx] = newPostMeta
break
}
}
if !_found {
allPostMetas = append(allPostMetas, newPostMeta)
}
}
return allPostMetas, founds
}
var DUP_THRESHOLD_RATE = 3
func ProcesserWorker(i int, tracker *savewebtracker.Tracker) {
Logger.Println("[START] ProcesserWorker", i)
defer Logger.Println("[STOP] ProcesserWorker", i, " exited...")
@ -83,7 +107,13 @@ func ProcesserWorker(i int, tracker *savewebtracker.Tracker) {
Logger.Panicln(head, err)
}
all_postMetas := []cnblogs_api.PostMeta{}
dups_found := 0
for page := 1; ; page++ {
if dups_found > len(all_postMetas)*DUP_THRESHOLD_RATE {
Logger.Println(head, "Dups found", dups_found, "exceeds the threshold", DUP_THRESHOLD_RATE, ", break")
break
}
Logger.Println(head, "Processing", blogURI, "page:", page, "Got:", len(all_postMetas))
htmlBody, statusCode, err := cnblogs_api.GetBlogHomepage(tracker.HTTP_client, blogURI, page)
if err != nil {
@ -104,7 +134,9 @@ func ProcesserWorker(i int, tracker *savewebtracker.Tracker) {
break
}
Logger.Println(head, "Got", postMetas)
all_postMetas = append(all_postMetas, postMetas...)
_all_postMetas, _founds_in_the_page := dedup_append(all_postMetas, postMetas)
dups_found += _founds_in_the_page
all_postMetas = _all_postMetas
}
items := []savewebtracker.Item{}
@ -170,7 +202,7 @@ func ShowStatus(t *savewebtracker.Tracker) {
}
func main() {
tracker := savewebtracker.GetTracker(project_id, "0.2", savewebtracker.Archivist())
tracker := savewebtracker.GetTracker(project_id, "0.3.0", savewebtracker.Archivist())
tracker.PING_client = GetRetryableHttpClient(10*time.Second, DEBUG)
// tracker.HTTP_client = GetRetryableHttpClient(10*time.Second, DEBUG)
tracker.SelectBestTracker().StartSelectTrackerBackground().StartFetchProjectBackground()