diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml new file mode 100644 index 0000000..8287813 --- /dev/null +++ b/.gitea/workflows/release.yml @@ -0,0 +1,41 @@ +name: Gitea Actions Demo +run-name: ${{ gitea.actor }} build goπŸš€ +on: + release: + types: [published] + +jobs: + Explore-Gitea-Actions: + runs-on: ubuntu-latest + steps: + - run: echo "πŸŽ‰ The job was automatically triggered by a ${{ gitea.event_name }} event." + - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by Gitea!" + - run: echo "πŸ”Ž The name of your branch is ${{ gitea.ref }} and your repository is ${{ gitea.repository }}." + - name: Check out repository code + uses: actions/checkout@v4 + - run: echo "πŸ’‘ The ${{ gitea.repository }} repository has been cloned to the runner." + - run: echo "πŸ–₯️ The workflow is now ready to test your code on the runner." + - name: List files in the repository + run: | + ls ${{ gitea.workspace }} + - run: echo "🍏 This job's status is ${{ job.status }}." + releases-matrix: + name: Release Go Binary + runs-on: ubuntu-latest + strategy: + matrix: + # build and publish in parallel: linux/386, linux/amd64, linux/arm64, windows/386, windows/amd64, darwin/amd64, darwin/arm64 + goos: [linux, windows, darwin] + goarch: ["386", amd64, arm64] + exclude: + - goarch: "386" + goos: darwin + - goarch: arm64 + goos: windows + steps: + - uses: actions/checkout@v4 + - uses: wangyoucao577/go-release-action@v1 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fee9217 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.conf diff --git a/cmd/huashijie_work_go/huashijie_work_go.go b/cmd/huashijie_work_go/huashijie_work_go.go new file mode 100644 index 0000000..298ea64 --- /dev/null +++ b/cmd/huashijie_work_go/huashijie_work_go.go @@ -0,0 +1,179 @@ +package main + +import ( + "encoding/json" + "fmt" + "log" + "net/http" + "os" + "os/signal" + "sync" + "time" + + "strconv" + + huashijie_api "git.saveweb.org/saveweb/huashijie_work_go/pkg" + savewebtracker "git.saveweb.org/saveweb/saveweb_tracker/src/saveweb_tracker" + "github.com/hashicorp/go-retryablehttp" +) + +var BASE_CONCURRENCY = 10 +var WITH_DELAY = true + +func init() { + if os.Getenv("BASE_CONCURRENCY") != "" { + Logger.Println("BASE_CONCURRENCY:", os.Getenv("BASE_CONCURRENCY")) + BASE_CONCURRENCY, _ = strconv.Atoi(os.Getenv("BASE_CONCURRENCY")) + } + if os.Getenv("NO_WITH_DELAY") != "" { + Logger.Println("NO_WITH_DELAY:", os.Getenv("NO_WITH_DELAY")) + WITH_DELAY = false + } +} + +var tasks_chan = make(chan savewebtracker.Task, BASE_CONCURRENCY) +var Interrupted = false +var WaitClaimWorker sync.WaitGroup +var WaitProcesserWorker sync.WaitGroup + +// 2024/06/08 16:22:36 [huashijie_work] ... +var Logger = log.New(os.Stdout, "[huashijie_work] ", log.Ldate|log.Ltime|log.Lmsgprefix) + +// ClaimTask εΉΆζŠŠδ»»εŠ‘ζ”Ύε…₯ task_chan +func claimWorker(i int, tracker *savewebtracker.Tracker) { + for { + if Interrupted { + Logger.Println("[STOP] ClaimWorker", i, "exitting...") + WaitClaimWorker.Done() + return + } + task := tracker.ClaimTask(WITH_DELAY) + if task == nil { + notask_sleep := max( + time.Duration(tracker.Project().Client.ClaimTaskDelay)*10*time.Second, + time.Duration(10)*time.Second, + ) + Logger.Println("No task to claim, sleep", notask_sleep) + time.Sleep(notask_sleep) + continue + } + Logger.Println("Claimed task", task) + tasks_chan <- *task + } +} + +func ProcesserWorker(i int, tracker *savewebtracker.Tracker) { + defer Logger.Println("[STOP] ProcesserWorker", i, " exited") + defer WaitProcesserWorker.Done() + for task := range tasks_chan { + Logger.Println("Processing task", task) + + // εœ¨θΏ™ε„Ώε€„η†δ»»εŠ‘ + body, r_status := huashijie_api.GetWorkDetailResponse(*tracker.HTTP_client, task.Id) + + if r_status != 200 { + Logger.Println("HTTP status code:", r_status, "body:", string(body)) + panic("HTTP status code: " + strconv.Itoa(r_status)) + } + + var r_json map[string]interface{} + + if err := json.Unmarshal(body, &r_json); err != nil { + Logger.Println("failed to parse JSON:", string(body), "error:", err) + panic("failed to parse JSON: " + string(body)) + } + TO_INSERT := false + item_status := 0 + + // check if 'status' in r_json + if status, ok := r_json["status"]; ok { + switch status := status.(type) { + case float64: + if status == 1 { + // OK + Logger.Println(r_json) + TO_INSERT = true + item_status = 1 + } else if status == 43 || status == 72 { + Logger.Println(r_json) + TO_INSERT = true + item_status = int(status) + } else { + panic("Unknown status: " + string(body)) + } + default: + panic("Unknown status: " + string(body)) + } + } else { + panic("Unknown response: " + string(body)) + } + + if TO_INSERT { + tracker.InsertItem(task, fmt.Sprintf("%d", item_status), "int", string(body)) + Logger.Println("Inserted item", task.Id) + tracker.UpdateTask(task.Id, task.Id_type, savewebtracker.StatusDONE) + Logger.Println("Updated task", task.Id) + } else { + panic("NotImplementedError: " + string(body)) + } + } +} + +func InterruptHandler() { + fmt.Println("Press Ctrl+C to exit") + interrupt_c := make(chan os.Signal, 1) + signal.Notify(interrupt_c, os.Interrupt) + for { + s := <-interrupt_c + Logger.Println("Interrupted by", s, "signal (Press Ctrl+C again to force exit)") + if Interrupted { + Logger.Println("Force exit") + os.Exit(1) + return + } + Interrupted = true + } +} + +func GetRetryableHttpClient(timeout time.Duration, debug bool) *http.Client { + retryClient := retryablehttp.NewClient() + retryClient.RetryMax = 3 + retryClient.RetryWaitMin = 1 * time.Second + retryClient.RetryWaitMax = 10 * time.Second + retryClient.HTTPClient.Timeout = timeout + if !debug { + retryClient.Logger = nil + } + standardClient := retryClient.StandardClient() // *http.Client + Logger.Println("standardClient.Timeout:", standardClient.Timeout) + return standardClient +} + +func main() { + tracker := savewebtracker.GetTracker("test", "1.0", savewebtracker.Archivist()+"-go-channel") + tracker.PING_client = GetRetryableHttpClient(10*time.Second, true) + tracker.HTTP_client = GetRetryableHttpClient(60*time.Second, true) + tracker.SelectBestTracker().StartSelectTrackerBackground().StartFetchProjectBackground() + + go InterruptHandler() + + for i := 0; i < BASE_CONCURRENCY; i++ { + go claimWorker(i, tracker) + WaitClaimWorker.Add(1) + go ProcesserWorker(i, tracker) + WaitProcesserWorker.Add(1) + } + + // wait for all claimWorker to finish + WaitClaimWorker.Wait() + Logger.Println("[STOP] All claimWorker done") + // close task_chan + close(tasks_chan) + Logger.Println("[STOP] task_chan closed") + // wait for all task_chan to finish + WaitProcesserWorker.Wait() + Logger.Println("[STOP] All ProcesserWorker done") + + Logger.Println("-- All done --") + +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..5c52384 --- /dev/null +++ b/go.mod @@ -0,0 +1,10 @@ +module git.saveweb.org/saveweb/huashijie_work_go + +go 1.22.3 + +require ( + git.saveweb.org/saveweb/saveweb_tracker v0.1.7 + github.com/hashicorp/go-retryablehttp v0.7.7 +) + +require github.com/hashicorp/go-cleanhttp v0.5.2 // indirect diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..48c7ea4 --- /dev/null +++ b/go.sum @@ -0,0 +1,24 @@ +git.saveweb.org/saveweb/saveweb_tracker v0.1.7 h1:j3OtEPNbK2sKYoYOE5nl3GD+4+9gozu4cRXLRYjJLp8= +git.saveweb.org/saveweb/saveweb_tracker v0.1.7/go.mod h1:p891f4fshoA/Wiwmey23f2xJ9sKNEZwd5kmzG6lobik= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/huashijie_api.go b/pkg/huashijie_api.go new file mode 100644 index 0000000..de469de --- /dev/null +++ b/pkg/huashijie_api.go @@ -0,0 +1,56 @@ +package huashijie_api + +import ( + "fmt" + "io" + "math/rand" + "net/http" +) + +var XIAOMI_MODELS = []string{ + "M2001J2E", + "M1810E5GG", + "2304FPN6DG", + "23127PN0CC", +} + +func GetWorkDetailResponse(client http.Client, work_id string) ([]byte, int) { + req, err := http.NewRequest("GET", "http://app.huashijie.art/api/work/detailV2", nil) + if err != nil { + panic(err) + } + q := req.URL.Query() + q.Add("visitorId", "-1") + q.Add("workId", work_id) + q.Add("cur_user_id", "-1") + q.Add("platform", "android") + os_version := rand.Intn(34-23) + 23 + q.Add("os_version", fmt.Sprintf("%d", os_version)) + q.Add("version_code", "224") + q.Add("device_brand", "Xiaomi") + device_model := XIAOMI_MODELS[rand.Intn(len(XIAOMI_MODELS))] + q.Add("device_model", device_model) + q.Add("token", "") + q.Add("channel", "main") + + headers := map[string][]string{ + "Referer": {"*.painterclub.cn", "*.pandapaint.net", "*.huashijie.art"}, + "User-Agent": {"okhttp/3.12.0"}, + } + for k, v := range headers { + req.Header[k] = v + } + req.URL.RawQuery = q.Encode() + + resp, err := client.Do(req) + if err != nil { + panic(err) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + panic(err) + } + return body, resp.StatusCode + +}