diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..faca9c4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.vscode/ +__pycache__/ +data/ +__debug_bin* \ No newline at end of file diff --git a/README.md b/README.md index b7093d0..9a7748f 100644 --- a/README.md +++ b/README.md @@ -1 +1,19 @@ -# Solar Tacker +# Solar Tracker + +## Why name Solar? + +Archive Team uses [ArchiveTeam/Universal-tracker](https://github.com/ArchiveTeam/Universal-tracker). +The solar system is insignificant compared to the universe. +Compared to ArchiveTeam, we don't have a lot of projects to track, so... "Solar Tracker" here is! + +## Tracker Configuration + +```bash +MONGODB_URI=mongodb://aa:bb@localhost:27017,s2:27017,s3:27017/?replicaSet=rs0 # mongodb uri +GIN_MODE=release # set to release to disable debug mode +PORT=8080 # port to listen, default 8080 +``` + +## Project configuration + +[projects.go](./projects.go) \ No newline at end of file diff --git a/api_v1.go b/api_v1.go new file mode 100644 index 0000000..9fd7ef6 --- /dev/null +++ b/api_v1.go @@ -0,0 +1,297 @@ +package main + +import ( + "context" + "fmt" + "strconv" + "time" + + "github.com/gin-gonic/gin" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" +) + +const DEFAULT_DOC_ID_NAME = "id" + +func v1_project(c *gin.Context) { + identifier := c.Param("identifier") + if !is_safe_sting(identifier) { + c.JSON(400, gin.H{"error": "Invalid identifier"}) + return + } + project := GetProject(identifier) + if project == nil { + c.JSON(404, gin.H{ + "error": fmt.Sprintf("Project %s not found", identifier), + }) + return + } + c.JSON(200, project) +} + +func v1_projects(c *gin.Context) { + projects := GetProjects() + pub_projects := []Project{} + show_private := c.Query("show_private") + if show_private != "" { + c.JSON(200, projects) + return + } + + for _, project := range projects { + if project.Status.Public { + pub_projects = append(pub_projects, project) + } + } + c.JSON(200, pub_projects) +} + +func ClaimTask(queue *mongo.Collection, from_status string, archivist string) *primitive.M { + filter := bson.M{"status": from_status} + update := bson.M{ + "$set": bson.M{ + "status": "PROCESSING", + "archivist": archivist, + "claimed_at": primitive.NewDateTimeFromTime(time.Now().UTC()), + "updated_at": primitive.NewDateTimeFromTime(time.Now().UTC()), + }} + + var task primitive.M + err := queue.FindOneAndUpdate(context.TODO(), filter, update).Decode(&task) + if err != nil { + if err == mongo.ErrNoDocuments { + return nil + } + panic(err) + } + return &task +} + +func v1_claim_task(c *gin.Context) { + identifier := c.Param("identifier") + client_version := c.Param("client_version") + archivist := c.Param("archivist") + if is_safe_sting(identifier) && is_safe_sting(archivist) { + // OK + } else { + c.JSON(400, gin.H{"error": "Invalid identifier or archivist"}) + return + } + + project := GetProject(identifier) + if project == nil { + c.JSON(404, gin.H{ + "error": fmt.Sprintf("Project %s not found", identifier), + }) + return + } + // 暂停后不再接受新的 claim_task 请求。 + if project.Status.Paused { + c.JSON(400, gin.H{ + "error": "Project paused", + }) + return + } + if client_version != project.Client.Version { + c.JSON(400, gin.H{ + "error": "Client version not supported", + "msg": fmt.Sprintf("Please update to version %s", project.Client.Version), + }) + return + } + + db := mongoClient.Database(project.Mongodb.DbName) + queue := db.Collection(project.Mongodb.QueueCollection) + + task := ClaimTask(queue, "TODO", archivist) + if task == nil { + c.JSON(404, gin.H{ + "error": "No task available", + }) + return + } + c.JSON(200, task) +} + +func v1_update_task(c *gin.Context) { + identifier := c.Param("identifier") + client_version := c.Param("client_version") + archivist := c.Param("archivist") + task_id_str := c.Param("task_id") + + status := c.PostForm("status") + task_id_type := c.PostForm("task_id_type") + + if is_safe_sting(identifier) && is_safe_sting(archivist) { + // OK + } else { + c.JSON(400, gin.H{"error": "Invalid parameter or query string"}) + return + } + + project := GetProject(identifier) + if project == nil { + c.JSON(404, gin.H{ + "error": fmt.Sprintf("Project %s not found", identifier), + }) + return + } + if client_version != project.Client.Version { + c.JSON(400, gin.H{ + "error": "Client version not supported", + "msg": fmt.Sprintf("Please update to version %s", project.Client.Version), + }) + return + } + + // 为了兼容 lowapk_v2 存档项目。它数据库用的 feed_id 而不是 id + var doc_id_name string + if project.Mongodb.CustomDocIDName != "" { + doc_id_name = project.Mongodb.CustomDocIDName + } else { + doc_id_name = DEFAULT_DOC_ID_NAME + } + + db := mongoClient.Database(project.Mongodb.DbName) + queue := db.Collection(project.Mongodb.QueueCollection) + + var filter bson.M + if task_id_type == "int" { + task_id, _ := strconv.ParseInt(task_id_str, 10, 64) + filter = bson.M{doc_id_name: task_id} + } else if task_id_type == "str" { + filter = bson.M{doc_id_name: task_id_str} + } else { + c.JSON(400, gin.H{"error": "Invalid task_id_type"}) + return + } + + update := bson.M{ + "$set": bson.M{ + "status": status, + "archivist": archivist, + "updated_at": primitive.NewDateTimeFromTime(time.Now().UTC()), + }} + + var updated_doc bson.M + err := queue.FindOneAndUpdate(context.TODO(), filter, update).Decode(&updated_doc) + if err != nil { + if err == mongo.ErrNoDocuments { + c.JSON(404, gin.H{ + "error": "Task not found", + }) + return + } + panic(err) + } + + c.JSON(200, gin.H{ + "_id": updated_doc["_id"], + "msg": "Task updated successfully", + }) +} + +func v1_insert_item(c *gin.Context) { + identifier := c.Param("identifier") + client_version := c.Param("client_version") + archivist := c.Param("archivist") + item_id_str := c.Param("item_id") + + item_id_type := c.PostForm("item_id_type") // str, int + item_status := c.PostForm("item_status") // item status + item_status_type := c.PostForm("item_status_type") // None, str, int + payload := c.PostForm("payload") // Any JSON string + + if is_safe_sting(identifier) && is_safe_sting(archivist) { + // OK + } else { + c.JSON(400, gin.H{"error": "Invalid parameter or query string"}) + return + } + + project := GetProject(identifier) + if project == nil { + c.JSON(404, gin.H{ + "error": fmt.Sprintf("Project %s not found", identifier), + }) + return + } + if client_version != project.Client.Version { + c.JSON(400, gin.H{ + "error": "Client version not supported", + "msg": fmt.Sprintf("Please update to version %s", project.Client.Version), + }) + return + } + + db := mongoClient.Database(project.Mongodb.DbName) + item_collection := db.Collection(project.Mongodb.ItemCollection) + + var doc_id_name string + if project.Mongodb.CustomDocIDName != "" { + doc_id_name = project.Mongodb.CustomDocIDName + } else { + doc_id_name = DEFAULT_DOC_ID_NAME + } + + document := bson.M{} + // id + if item_id_type == "str" { + document[doc_id_name] = item_id_str + } else if item_id_type == "int" { + item_id_int, err := strconv.ParseInt(item_id_str, 10, 64) + if err != nil { + c.JSON(400, gin.H{"error": "Invalid item_id"}) + return + } + document[doc_id_name] = item_id_int + } else { + c.JSON(400, gin.H{"error": "Invalid task_id_type"}) + return + } + // status + if item_status_type == "str" { + document["status"] = item_status + } else if item_status_type == "int" { + status, err := strconv.ParseInt(item_status, 10, 64) + if err != nil { + c.JSON(400, gin.H{"error": "Invalid item_status"}) + return + } + document["status"] = status + } else if item_status_type == "None" { + document["status"] = nil + } else { + c.JSON(400, gin.H{"error": "Invalid status_type"}) + return + } + // payload + var payload_BSON primitive.M + err := bson.UnmarshalExtJSON([]byte(payload), true, &payload_BSON) + if err != nil { + c.JSON(400, gin.H{"error": "Invalid JSON payload"}) + panic(err) + } + document["payload"] = payload_BSON + + // do insert + result, err := item_collection.InsertOne(context.TODO(), document) + if err != nil { + if mongo.IsDuplicateKeyError(err) { + c.JSON(500, gin.H{"error": "Failed to insert item, duplicate key"}) + return + } + c.JSON(500, gin.H{"error": "Failed to insert item"}) + panic(err) + } + if result.InsertedID == nil { + c.JSON(500, gin.H{"error": "Failed to insert item"}) + panic("Failed to insert item") + } + + c.JSON(200, gin.H{ + "_id": result.InsertedID, + "msg": "Item inserted successfully", + }) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..191c962 --- /dev/null +++ b/go.mod @@ -0,0 +1,43 @@ +module github.com/saveweb/solar_tracker + +go 1.22.2 + +require ( + github.com/gin-gonic/gin v1.9.1 + go.mongodb.org/mongo-driver v1.15.0 +) + +require ( + github.com/bytedance/sonic v1.9.1 // indirect + github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect + github.com/gabriel-vasile/mimetype v1.4.2 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.14.0 // indirect + github.com/goccy/go-json v0.10.2 // indirect + github.com/golang/snappy v0.0.1 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.13.6 // indirect + github.com/klauspost/cpuid/v2 v2.2.4 // indirect + github.com/leodido/go-urn v1.2.4 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect + github.com/pelletier/go-toml/v2 v2.0.8 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.11 // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/scram v1.1.2 // indirect + github.com/xdg-go/stringprep v1.0.4 // indirect + github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect + golang.org/x/arch v0.3.0 // indirect + golang.org/x/crypto v0.17.0 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/sync v0.1.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/protobuf v1.30.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..a46484c --- /dev/null +++ b/go.sum @@ -0,0 +1,128 @@ +github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= +github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= +github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= +github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= +github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= +github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= +github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= +github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= +github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= +github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= +github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.mongodb.org/mongo-driver v1.15.0 h1:rJCKC8eEliewXjZGf0ddURtl7tTVy1TK3bfl0gkUSLc= +go.mongodb.org/mongo-driver v1.15.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= +golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/main.go b/main.go new file mode 100644 index 0000000..5fef517 --- /dev/null +++ b/main.go @@ -0,0 +1,60 @@ +package main + +import ( + "context" + "fmt" + "os" + + "github.com/gin-gonic/gin" + + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +var MONGODB_URI string = os.Getenv("MONGODB_URI") + +var mongoClient *mongo.Client + +func connect_to_mongodb() { + fmt.Println("Connecting to MongoDB...") + fmt.Println("MONGODB_URI: len=", len(MONGODB_URI)) + serverAPI := options.ServerAPI(options.ServerAPIVersion1) + opts := options.Client().ApplyURI(MONGODB_URI).SetServerAPIOptions(serverAPI) + + client, err := mongo.Connect(context.TODO(), opts) + if err != nil { + panic(err) + } + err = client.Ping(context.TODO(), nil) + if err != nil { + panic(err) + } + mongoClient = client + fmt.Println("Connected to MongoDB!") +} + +func ping(c *gin.Context) { + c.JSON(200, gin.H{ + "message": "pong", + }) +} + +func init() { + connect_to_mongodb() +} +func main() { + r := gin.Default() + + r.GET("/ping", ping) + r.HEAD("/ping", ping) + + v1_tracker := r.Group("/v1") + { + v1_tracker.POST("/projects", v1_projects) + v1_tracker.POST("/project/:identifier", v1_project) + v1_tracker.POST("/project/:identifier/:client_version/:archivist/claim_task", v1_claim_task) + v1_tracker.POST("/project/:identifier/:client_version/:archivist/update_task/:task_id", v1_update_task) + v1_tracker.POST("/project/:identifier/:client_version/:archivist/insert_item/:item_id", v1_insert_item) + } + r.Run() +} diff --git a/projects.go b/projects.go new file mode 100644 index 0000000..587f1dd --- /dev/null +++ b/projects.go @@ -0,0 +1,96 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" +) + +type Meta struct { + Identifier string `json:"identifier"` + Slug string `json:"slug"` // 一句话项目说明 + Icon string `json:"icon"` // 图标 URL + Deadline string `json:"deadline"` // 截止日期，没有格式要求 +} + +type Status struct { + Public bool `json:"public"` // 是否公开。（不在 /projects 列表中列出，但 /project 可以请求） + Paused bool `json:"paused"` // 是否暂停。暂停后不再接受新的 claim_task 请求。但是仍可 update_task 和 insert_item +} + +type Client struct { + Version string `json:"version"` // 推荐 "大版本.小版本"。版本不对会拒绝各种请求。 + ClaimTaskDelay float32 `json:"claim_task_delay"` // claim_task 之后多久才能再次 claim_task +} + +type Mongodb struct { + DbName string `json:"db_name"` + ItemCollection string `json:"item_collection"` + QueueCollection string `json:"queue_collection"` + CustomDocIDName string `json:"custom_doc_id_name"` // CustomIDName 目前只是为了兼容 lowapk_v2 存档项目而留的字段。它用的 feed_id 而不是 id + // 计划未来等转换好数据库 scheme 之后就弃用。 +} + +type Project struct { + Meta Meta `json:"meta"` + Status Status `json:"status"` + Client Client `json:"client"` + Mongodb Mongodb `json:"mongodb"` +} + +const PROJECTS_DIR = "data/projects" +const PROJECT_FILE = "project.json" + +func GetProjectsName() []string { + projects := []string{} + // {projects_dir}/{project}/project.json + + dirs, _ := os.ReadDir(PROJECTS_DIR) + for _, dir := range dirs { + if dir.IsDir() { + projects = append(projects, dir.Name()) + } else { + fmt.Printf("%s is not a directory, skipping...\n", dir.Name()) + continue + } + } + return projects +} + +func GetProject(project_name string) *Project { + project_file_path := fmt.Sprintf("%s/%s/%s", PROJECTS_DIR, project_name, PROJECT_FILE) + project_data, _ := os.ReadFile(project_file_path) + project := Project{} + err := json.Unmarshal(project_data, &project) + if err != nil { + return nil + } + return &project +} + +func GetProjects() []Project { + projects := []Project{} + // {projects_dir}/{project}/project.json + + dirs, err := os.ReadDir(PROJECTS_DIR) + if err != nil { + panic(err) + } + for _, dir := range dirs { + if dir.IsDir() { + project_file_path := fmt.Sprintf("%s/%s/%s", PROJECTS_DIR, dir.Name(), PROJECT_FILE) + project_data, _ := os.ReadFile(project_file_path) + + project := Project{} + err = json.Unmarshal(project_data, &project) + if err != nil { + panic(err) + } + projects = append(projects, project) + } else { + fmt.Printf("%s is not a directory, skipping...\n", dir.Name()) + continue + } + } + return projects +} diff --git a/util.go b/util.go new file mode 100644 index 0000000..a45af5b --- /dev/null +++ b/util.go @@ -0,0 +1,13 @@ +package main + +import "regexp" + +// only keep a-z A-Z 0-9 and _ -, remove other characters +func safe_string(s string) string { + return regexp.MustCompile(`[^a-zA-Z0-9_\-]`).ReplaceAllString(s, "") +} + +// a-z A-Z 0-9 and _ - are safe +func is_safe_sting(s string) bool { + return (s != "" && s == safe_string(s)) +}