panicDelyRecoverExit->main()

EnsureConnection: check GetWorkDetailResponse

.gitea/workflows/release.yml

@@ -1,6 +1,6 @@
-name: Gitea Actions Demo
-run-name: ${{ gitea.actor }} build go🚀
-on: 
+name: Gitea Go Release Actions
+run-name: ${{ gitea.actor }} go🚀
+on:
   push:
     tags:
       - '*'

.gitignore

@@ -1 +1,2 @@
 *.conf
+huashijie_work_go

cmd/huashijie_work_go/huashijie_work_go.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
+	"runtime/debug"
 	"sync"
 	"time"
 
@@ -17,36 +18,70 @@ import (
 	"github.com/hashicorp/go-retryablehttp"
 )
 
-var BASE_CONCURRENCY = 10
+var BASE_CONCURRENCY = 2
 var WITH_DELAY = true
 
-func init() {
-	if os.Getenv("BASE_CONCURRENCY") != "" {
-		Logger.Println("BASE_CONCURRENCY:", os.Getenv("BASE_CONCURRENCY"))
-		BASE_CONCURRENCY, _ = strconv.Atoi(os.Getenv("BASE_CONCURRENCY"))
-	}
-	if os.Getenv("NO_WITH_DELAY") != "" {
-		Logger.Println("NO_WITH_DELAY:", os.Getenv("NO_WITH_DELAY"))
-		WITH_DELAY = false
-	}
-}
-
 var tasks_chan = make(chan savewebtracker.Task, BASE_CONCURRENCY)
 var Interrupted = false
 var WaitClaimWorker sync.WaitGroup
 var WaitProcesserWorker sync.WaitGroup
 
-// 2024/06/08 16:22:36 [huashijie_work] ...
-var Logger = log.New(os.Stdout, "[huashijie_work] ", log.Ldate|log.Ltime|log.Lmsgprefix)
+var project_id = "huashijie_work"
+
+var Logger *log.Logger
+var DEBUG = false
+
+func init() {
+	if os.Getenv("BASE_CONCURRENCY") != "" {
+		fmt.Println("BASE_CONCURRENCY:", os.Getenv("BASE_CONCURRENCY"))
+		BASE_CONCURRENCY, _ = strconv.Atoi(os.Getenv("BASE_CONCURRENCY"))
+	}
+	if os.Getenv("NO_WITH_DELAY") != "" {
+		fmt.Println("NO_WITH_DELAY:", os.Getenv("NO_WITH_DELAY"))
+		WITH_DELAY = false
+	}
+	if os.Getenv("DEBUG") != "" {
+		DEBUG = true
+	}
+	if os.Getenv("HSJ_ENDPOINT") == "pandapaint" {
+		project_id = "huashijie_work_pandapaint"
+	}
+	// 2024/06/08 16:22:36 [huashijie_work] ...
+	Logger = log.New(os.Stdout, "["+project_id+"] ", log.Ldate|log.Ltime|log.Lmsgprefix)
+}
+
+var custom_delay_lock = sync.Mutex{}
+
+func panicDelyRecoverExit() {
+	if r := recover(); r != nil {
+		Logger.Println("Panic:", r)
+		Logger.Println("debug.Stack():", string(debug.Stack()))
+		Logger.Println("Sleep 60s before exit...")
+		time.Sleep(60 * time.Second)
+		os.Exit(1)
+	}
+}
 
 // ClaimTask 并把任务放入 task_chan
 func claimWorker(i int, tracker *savewebtracker.Tracker) {
+	Logger.Println("[START] ClaimWorker", i)
+	defer Logger.Println("[STOP] ClaimWorker", i, " exited...")
+	defer WaitClaimWorker.Done()
+	defer panicDelyRecoverExit()
 	for {
 		if Interrupted {
-			Logger.Println("[STOP] ClaimWorker", i, "exitting...")
-			WaitClaimWorker.Done()
 			return
 		}
+		if os.Getenv("CUSTOM_DELAY") != "" {
+			custom_delay_lock.Lock()
+			// xxxms
+			custom_delay, _ := time.ParseDuration(os.Getenv("CUSTOM_DELAY"))
+			Logger.Println("Custom delay:", custom_delay, "...")
+			time.Sleep(custom_delay)
+			custom_delay_lock.Unlock()
+
+			WITH_DELAY = false
+		}
 		task := tracker.ClaimTask(WITH_DELAY)
 		if task == nil {
 			notask_sleep := max(
@@ -57,16 +92,18 @@ func claimWorker(i int, tracker *savewebtracker.Tracker) {
 			time.Sleep(notask_sleep)
 			continue
 		}
-		Logger.Println("Claimed task", task)
+		Logger.Println("Claimed task", task.Id)
 		tasks_chan <- *task
 	}
 }
 
 func ProcesserWorker(i int, tracker *savewebtracker.Tracker) {
-	defer Logger.Println("[STOP] ProcesserWorker", i, " exited")
+	Logger.Println("[START] ProcesserWorker", i)
+	defer Logger.Println("[STOP] ProcesserWorker", i, " exited...")
 	defer WaitProcesserWorker.Done()
+	defer panicDelyRecoverExit()
 	for task := range tasks_chan {
-		Logger.Println("Processing task", task)
+		Logger.Println("Processing task", task.Id)
 
 		// 在这儿处理任务
 		body, r_status := huashijie_api.GetWorkDetailResponse(*tracker.HTTP_client, task.Id)
@@ -91,7 +128,7 @@ func ProcesserWorker(i int, tracker *savewebtracker.Tracker) {
 			case float64:
 				if status == 1 {
 					// OK
-					Logger.Println(r_json)
+					Logger.Println("len(body):", len(body), "->", string(body[:80]), "...")
 					TO_INSERT = true
 					item_status = 1
 				} else if status == 43 || status == 72 {
@@ -149,13 +186,35 @@ func GetRetryableHttpClient(timeout time.Duration, debug bool) *http.Client {
 	return standardClient
 }
 
+func ShowStatus(t *savewebtracker.Tracker) {
+	for {
+		project_json, err := json.Marshal(t.Project())
+		if err != nil {
+			panic(err)
+		}
+		Logger.Println("Project:", string(project_json))
+		time.Sleep(60 * time.Second)
+	}
+}
+
 func main() {
-	tracker := savewebtracker.GetTracker("test", "1.0", savewebtracker.Archivist()+"-go-channel")
-	tracker.PING_client = GetRetryableHttpClient(10*time.Second, true)
-	tracker.HTTP_client = GetRetryableHttpClient(60*time.Second, true)
+	defer panicDelyRecoverExit()
+	tracker := savewebtracker.GetTracker(project_id, "0.5", savewebtracker.Archivist())
+	tracker.PING_client = GetRetryableHttpClient(10*time.Second, DEBUG)
+	tracker.HTTP_client = GetRetryableHttpClient(10*time.Second, DEBUG)
 	tracker.SelectBestTracker().StartSelectTrackerBackground().StartFetchProjectBackground()
 
 	go InterruptHandler()
+	go ShowStatus(tracker)
+
+	time.Sleep(3 * time.Second)
+	Logger.Println("force sleep 10s before start...")
+	time.Sleep(7 * time.Second)
+	Logger.Println("start...")
+
+	huashijie_api.EnsureConnection(*tracker.HTTP_client)
+
+	Logger.Println("-- Start --")
 
 	for i := 0; i < BASE_CONCURRENCY; i++ {
 		go claimWorker(i, tracker)

go.mod

@@ -1,9 +1,9 @@
 module git.saveweb.org/saveweb/huashijie_work_go
 
-go 1.22.3
+go 1.22.4
 
 require (
-	git.saveweb.org/saveweb/saveweb_tracker v0.1.7
+	git.saveweb.org/saveweb/saveweb_tracker v0.1.12
 	github.com/hashicorp/go-retryablehttp v0.7.7
 )
 

go.sum

@@ -1,5 +1,5 @@
-git.saveweb.org/saveweb/saveweb_tracker v0.1.7 h1:j3OtEPNbK2sKYoYOE5nl3GD+4+9gozu4cRXLRYjJLp8=
-git.saveweb.org/saveweb/saveweb_tracker v0.1.7/go.mod h1:p891f4fshoA/Wiwmey23f2xJ9sKNEZwd5kmzG6lobik=
+git.saveweb.org/saveweb/saveweb_tracker v0.1.12 h1:zBYkMjABF5wwvSHZI9t3cVUjU0rhFFZJh0dFE0W59Nw=
+git.saveweb.org/saveweb/saveweb_tracker v0.1.12/go.mod h1:p891f4fshoA/Wiwmey23f2xJ9sKNEZwd5kmzG6lobik=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=

pkg/huashijie_api.go

@@ -1,46 +1,162 @@
 package huashijie_api
 
 import (
+	"encoding/json"
 	"fmt"
 	"io"
+	"log"
 	"math/rand"
 	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"time"
 )
 
 var XIAOMI_MODELS = []string{
-	"M2001J2E",
+	"2107119DC",
+	"23116PN5BC",
+	"23124RN87C",
+	"24031PN0DC",
+	"24072PX77C",
+	"24074RPD2C",
+	"M1804E4A",
 	"M1810E5GG",
-	"2304FPN6DG",
-	"23127PN0CC",
+	"M2001J1C",
+	"M2002J9E",
+	"MDE5",
 }
 
-func GetWorkDetailResponse(client http.Client, work_id string) ([]byte, int) {
-	req, err := http.NewRequest("GET", "http://app.huashijie.art/api/work/detailV2", nil)
+var Logger = log.New(os.Stdout, "[huashijie_api] ", log.Ldate|log.Ltime|log.Lmsgprefix)
+
+func EnsureConnection(client http.Client) {
+	// https://app.huashijie.art/api/update/checkUpdate
+	req, err := http.NewRequest("GET", "https://app.huashijie.art/api/update/checkUpdate", nil)
 	if err != nil {
 		panic(err)
 	}
-	q := req.URL.Query()
-	q.Add("visitorId", "-1")
-	q.Add("workId", work_id)
-	q.Add("cur_user_id", "-1")
-	q.Add("platform", "android")
-	os_version := rand.Intn(34-23) + 23
-	q.Add("os_version", fmt.Sprintf("%d", os_version))
-	q.Add("version_code", "224")
-	q.Add("device_brand", "Xiaomi")
+	// req.Header.Add("Referer", "https://app.huashijie.art/")
+	req.Header.Add("User-Agent", "Dalvik/2.1.0 (Linux; U; Android 9; MI 8 SE MIUI/V12.0.2.0.PEBCNXM)")
+
+	resp, err := client.Do(req)
+
+	if err != nil {
+		panic(err)
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		panic(err)
+	}
+	text := string(body)
+	Logger.Println(text)
+	if !strings.Contains(text, "update_ver_code") {
+		panic("NotImplementedError: " + text)
+	}
+
+	// 125277547
+	body, r_status := GetWorkDetailResponse(client, "129107778")
+
+	if r_status != 200 {
+		Logger.Println("HTTP status code:", r_status, "body:", string(body))
+		panic("HTTP status code: " + strconv.Itoa(r_status))
+	}
+
+	var r_json map[string]interface{}
+
+	if err := json.Unmarshal(body, &r_json); err != nil {
+		Logger.Println("failed to parse JSON:", string(body), "error:", err)
+		panic("failed to parse JSON: " + string(body))
+	}
+
+	if !strings.Contains(string(body), "要你管啦离人家远远") {
+		Logger.Println("JSON:", string(body))
+		Logger.Panicln("assertion failed: body not contains '要你管啦离人家远远'")
+	}
+
+	// check if 'status' in r_json
+	if status, ok := r_json["status"]; ok {
+		switch status := status.(type) {
+		case float64:
+			if status == 1 {
+				// OK
+				Logger.Println("len(body):", len(body), "->", string(body[:256]), "...OK")
+				return
+			}
+		}
+	}
+	Logger.Println("JSON:", string(body))
+	panic("assertion failed: status == 1")
+}
+
+var cur_fake_user_id = -1
+var cur_fake_user_left = time.Now()
+
+func GetWorkDetailResponse(client http.Client, work_id string) ([]byte, int) {
+	endpoint := "https://app.huashijie.art/"
+	if os.Getenv("HSJ_ENDPOINT") == "pandapaint" {
+		endpoint = "https://api.pandapaint.net/"
+	}
+	req, err := http.NewRequest("GET", endpoint+"api/work/detailV2", nil)
+	if err != nil {
+		panic(err)
+	}
+
+	visitorId := "-1"
+	token := ""
+	if os.Getenv("HSJ_USERID") != "" {
+		visitorId = os.Getenv("HSJ_USERID")
+		if visitorId == "random" {
+			if cur_fake_user_left.Before(time.Now()) { // rotate [1, 15] minutes
+				cur_fake_user_left = time.Now().Add(time.Duration(rand.Intn(15)) * time.Minute)
+				// random [30000000, 40000000]
+				cur_fake_user_id = rand.Intn(40000000-30000000) + 30000000
+			}
+			visitorId = fmt.Sprintf("%d", cur_fake_user_id)
+		}
+
+		// fake token = md5(visitorId + "huashijie")
+		// md5hash := md5.New()
+		// md5hash.Write([]byte(visitorId + "huashijie"))
+		// token = fmt.Sprintf("%x", md5hash.Sum(nil))
+
+		Logger.Println("USERID:", visitorId, "token:", token, "left", time.Until(cur_fake_user_left))
+	}
+
+	p := ""
+	p += "visitorId=" + visitorId
+	p += "&workId=" + work_id
+	p += "&cur_user_id=" + visitorId
+	p += "&platform=android"
+	os_version := rand.Intn(34-28) + 28
+	p += fmt.Sprintf("&os_version=%d", os_version)
+
+	if os.Getenv("HSJ_ENDPOINT") == "pandapaint" {
+		p += "&version_code=170"
+	} else {
+		p += "&version_code=247"
+	}
+
+	p += "&device_brand=xiaomi"
 	device_model := XIAOMI_MODELS[rand.Intn(len(XIAOMI_MODELS))]
-	q.Add("device_model", device_model)
-	q.Add("token", "")
-	q.Add("channel", "main")
+	p += "&device_model=" + device_model
+	p += "&token=" + token
+
+	if os.Getenv("HSJ_ENDPOINT") == "pandapaint" {
+		p += "&channel=Panda.main"
+	} else {
+		p += "&channel=main"
+	}
 
 	headers := map[string][]string{
-		"Referer":    {"*.painterclub.cn", "*.pandapaint.net", "*.huashijie.art"},
+		"Referer":    {endpoint},
 		"User-Agent": {"okhttp/3.12.0"},
 	}
 	for k, v := range headers {
 		req.Header[k] = v
 	}
-	req.URL.RawQuery = q.Encode()
+	// req.URL.RawQuery = q.Encode()
+	req.URL.RawQuery = p
 
 	resp, err := client.Do(req)
 	if err != nil {