diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..192edcd --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +.git +.idea +.env +uploads +tmp +data +postgres +*.db diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..3f2a460 --- /dev/null +++ b/.env.example @@ -0,0 +1,16 @@ +# Required: token for the X-Admin-Token header on /admin routes +ADMIN_TOKEN=change-me + +# Required: Postgres connection string +DATABASE_URL=postgres://kooltap:kooltap@localhost:5432/kooltap?sslmode=disable + +# Used by docker-compose.yaml for the bundled Postgres container +POSTGRES_PASSWORD=kooltap + +# Optional (defaults shown) +# PORT=3000 +# UPLOAD_DIR=uploads +# TMP_DIR=tmp +# MAX_UPLOAD_MB=300 +# Old SQLite database to import once on startup, if present +# SQLITE_IMPORT_PATH=data/updater.db diff --git a/.gitignore b/.gitignore index 00af6fb..5c43b0a 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,6 @@ cmake-build-*/ out/ uploads updater.db +tmp +data +postgres diff --git a/Dockerfile b/Dockerfile index 1f5b4ec..47ab6cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,37 +2,31 @@ FROM golang:1.26-alpine AS builder WORKDIR /app -RUN apk add --no-cache git gcc musl-dev bash - COPY go.mod go.sum ./ RUN go mod download COPY . . -ENV CGO_ENABLED=1 -ENV GIN_MODE=release +# All dependencies (pgx, the SQLite importer) are pure Go, so build a static binary +ENV CGO_ENABLED=0 -RUN go build -o app ./cmd/server +RUN go build -trimpath -ldflags="-s -w" -o app ./cmd/server -FROM alpine:latest +FROM alpine:3.22 WORKDIR /app -RUN apk add --no-cache ca-certificates tzdata +RUN apk add --no-cache ca-certificates tzdata \ + && adduser -D appuser \ + && mkdir -p /app/uploads /app/tmp /app/data \ + && chown -R appuser:appuser /app COPY --from=builder /app/app . -RUN mkdir -p /app/uploads -RUN mkdir -p /app/tmp -RUN mkdir -p /app/data - -RUN adduser -D appuser -RUN chown -R appuser:appuser /app - USER appuser ENV GIN_MODE=release EXPOSE 3000 -CMD ["./app"] \ No newline at end of file +CMD ["./app"] diff --git a/cmd/server/main.go b/cmd/server/main.go index 78f9896..841da66 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -1,9 +1,17 @@ package main import ( + "context" + "errors" "kooltap-updateserver/internal/db" "kooltap-updateserver/internal/routes" + "log" + "net/http" "os" + "os/signal" + "strconv" + "syscall" + "time" "github.com/gin-gonic/gin" "github.com/joho/godotenv" @@ -13,26 +21,99 @@ func main() { _ = godotenv.Load() - db.Connect() + dsn := os.Getenv("DATABASE_URL") + if dsn == "" { + log.Fatal("DATABASE_URL is required, e.g. postgres://user:pass@host:5432/kooltap?sslmode=disable") + } - os.MkdirAll("uploads", 0755) + database, err := db.Connect(dsn) + if err != nil { + log.Fatal(err) + } + + // One-time migration of the old file based database, if it is still mounted + if err := db.ImportSQLite(database, env("SQLITE_IMPORT_PATH", "data/updater.db")); err != nil { + log.Fatal(err) + } + + h := &routes.Handler{ + DB: database, + UploadDir: env("UPLOAD_DIR", "uploads"), + TmpDir: env("TMP_DIR", "tmp"), + MaxUpload: int64(envInt("MAX_UPLOAD_MB", 300)) << 20, + } + + for _, dir := range []string{h.UploadDir, h.TmpDir} { + if err := os.MkdirAll(dir, 0755); err != nil { + log.Fatal(err) + } + } r := gin.Default() - r.Static("/downloads", "./uploads") - - r.GET("/", func(c *gin.Context) { - c.JSON(200, gin.H{ - "status": "ok", - }) - }) + // No proxy headers are used for anything, so do not trust any + _ = r.SetTrustedProxies(nil) r.Use(func(c *gin.Context) { c.Header("Server", "KoolTap Updater") c.Next() }) - routes.Register(r) + r.Static("/downloads", h.UploadDir) - r.Run(":3000") + r.GET("/", func(c *gin.Context) { + sqlDB, err := database.DB() + if err == nil { + err = sqlDB.PingContext(c.Request.Context()) + } + if err != nil { + c.JSON(http.StatusServiceUnavailable, gin.H{"status": "database unavailable"}) + return + } + c.JSON(http.StatusOK, gin.H{"status": "ok"}) + }) + + h.Register(r) + + srv := &http.Server{ + Addr: ":" + env("PORT", "3000"), + Handler: r, + ReadHeaderTimeout: 10 * time.Second, + } + + go func() { + log.Printf("listening on %s", srv.Addr) + if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Fatal(err) + } + }() + + // Let in-flight uploads finish when the container is stopped + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + <-ctx.Done() + + log.Println("shutting down") + shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := srv.Shutdown(shutdownCtx); err != nil { + log.Printf("shutdown: %v", err) + } + if sqlDB, err := database.DB(); err == nil { + sqlDB.Close() + } +} + +func env(key, fallback string) string { + if v := os.Getenv(key); v != "" { + return v + } + return fallback +} + +func envInt(key string, fallback int) int { + if v, err := strconv.Atoi(os.Getenv(key)); err == nil && v > 0 { + return v + } + return fallback } diff --git a/docker-compose.coolify.yaml b/docker-compose.coolify.yaml index 2943782..fbc429f 100644 --- a/docker-compose.coolify.yaml +++ b/docker-compose.coolify.yaml @@ -6,14 +6,20 @@ services: restart: unless-stopped + depends_on: + postgres: + condition: service_healthy + environment: GIN_MODE: release # Generated by Coolify (see Environment Variables); needed in the X-Admin-Token header to upload releases ADMIN_TOKEN: ${SERVICE_PASSWORD_64_ADMINTOKEN} + DATABASE_URL: postgres://${SERVICE_USER_POSTGRES}:${SERVICE_PASSWORD_POSTGRES}@postgres:5432/${POSTGRES_DB:-kooltap}?sslmode=disable volumes: - kooltap_uploads:/app/uploads - kooltap_tmp:/app/tmp + # Only needed until the old SQLite database has been imported into Postgres on first start - kooltap_data:/app/data expose: @@ -25,7 +31,27 @@ services: timeout: 5s retries: 5 + postgres: + image: postgres:17-alpine + + restart: unless-stopped + + environment: + POSTGRES_USER: ${SERVICE_USER_POSTGRES} + POSTGRES_PASSWORD: ${SERVICE_PASSWORD_POSTGRES} + POSTGRES_DB: ${POSTGRES_DB:-kooltap} + + volumes: + - kooltap_postgres:/var/lib/postgresql/data + + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] + interval: 5s + timeout: 5s + retries: 10 + volumes: kooltap_uploads: kooltap_tmp: kooltap_data: + kooltap_postgres: diff --git a/docker-compose.yaml b/docker-compose.yaml index 6c5224b..ef2c6ab 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,5 +1,3 @@ -version: "3.9" - services: kooltap-updater: image: kooltap-updater:latest @@ -9,18 +7,46 @@ services: #ports: # - "3000:3000" + depends_on: + postgres: + condition: service_healthy + environment: GIN_MODE: release ADMIN_TOKEN: ${ADMIN_TOKEN} + DATABASE_URL: postgres://kooltap:${POSTGRES_PASSWORD}@kooltap-postgres:5432/kooltap?sslmode=disable volumes: - ./uploads:/app/uploads - ./tmp:/app/tmp + # Only needed until the old SQLite database has been imported into Postgres on first start - ./data:/app/data networks: - scoobydoo + postgres: + image: postgres:17-alpine + container_name: kooltap-postgres + restart: unless-stopped + + environment: + POSTGRES_USER: kooltap + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: kooltap + + volumes: + - ./postgres:/var/lib/postgresql/data + + healthcheck: + test: ["CMD-SHELL", "pg_isready -U kooltap -d kooltap"] + interval: 5s + timeout: 5s + retries: 10 + + networks: + - scoobydoo + networks: scoobydoo: - external: true \ No newline at end of file + external: true diff --git a/go.mod b/go.mod index e32ea0d..aa2228a 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,9 @@ require ( github.com/Masterminds/semver/v3 v3.5.0 github.com/gin-gonic/gin v1.12.0 github.com/glebarez/sqlite v1.11.0 + github.com/joho/godotenv v1.5.1 + github.com/shogo82148/androidbinary v1.0.5 + gorm.io/driver/postgres v1.6.3 gorm.io/gorm v1.31.2 ) @@ -24,9 +27,12 @@ require ( github.com/goccy/go-json v0.10.5 // indirect github.com/goccy/go-yaml v1.19.2 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.10.0 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect - github.com/joho/godotenv v1.5.1 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect @@ -37,13 +43,13 @@ require ( github.com/quic-go/qpack v0.6.0 // indirect github.com/quic-go/quic-go v0.59.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - github.com/shogo82148/androidbinary v1.0.5 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.3.1 // indirect go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect golang.org/x/arch v0.22.0 // indirect golang.org/x/crypto v0.48.0 // indirect golang.org/x/net v0.51.0 // indirect + golang.org/x/sync v0.19.0 // indirect golang.org/x/sys v0.41.0 // indirect golang.org/x/text v0.34.0 // indirect google.golang.org/protobuf v1.36.10 // indirect diff --git a/go.sum b/go.sum index a67d592..57e4544 100644 --- a/go.sum +++ b/go.sum @@ -42,6 +42,14 @@ github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbu github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0= +github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= @@ -81,6 +89,7 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= @@ -101,6 +110,8 @@ golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= @@ -112,6 +123,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.6.3 h1:bAn6O2pUa8LtpWEvL5NFU4+52Tfx8Ut7IVaIacCLcI0= +gorm.io/driver/postgres v1.6.3/go.mod h1:0c4fQA44XhOklXDkgtuKqysHCycTa5i9e3EIpDGCwXk= gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ= gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8= gorm.io/gorm v1.31.2 h1:3o8FXNo9v9S858gil+3LlZA1LkCOzgb4g5BL64FgaCo= diff --git a/internal/apk_util/apk.go b/internal/apk_util/apk.go index 8451bad..9e83e5a 100644 --- a/internal/apk_util/apk.go +++ b/internal/apk_util/apk.go @@ -10,22 +10,6 @@ type Info struct { VersionCode int32 } -// Version returns the versionName from an APK's manifest. -func Version(path string) (string, error) { - a, err := apk.OpenFile(path) - if err != nil { - return "", err - } - defer a.Close() - - v, err := a.Manifest().VersionName.String() - if err != nil { - return "", err - } - - return v, nil -} - // Read returns version name and version code from an APK's manifest. func Read(path string) (*Info, error) { a, err := apk.OpenFile(path) diff --git a/internal/db/db.go b/internal/db/db.go index fe0db2a..e613287 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -1,26 +1,77 @@ package db import ( + "context" + "fmt" "kooltap-updateserver/internal/models" "log" + "time" - "github.com/glebarez/sqlite" + "gorm.io/driver/postgres" "gorm.io/gorm" + "gorm.io/gorm/logger" ) -var DB *gorm.DB +// Connect opens the Postgres database and migrates the schema. +// It retries for a while because the database container may still be starting. +func Connect(dsn string) (*gorm.DB, error) { -func Connect() { + cfg := &gorm.Config{ + // Turns driver specific unique violations into gorm.ErrDuplicatedKey + TranslateError: true, + Logger: logger.Default.LogMode(logger.Warn), + } - db, err := gorm.Open(sqlite.Open("/app/data/updater.db"), &gorm.Config{}) + var ( + db *gorm.DB + err error + ) + deadline := time.Now().Add(60 * time.Second) + for { + db, err = gorm.Open(postgres.Open(dsn), cfg) + if err == nil { + err = ping(db) + } + if err == nil || time.Now().After(deadline) { + break + } + log.Printf("database not ready, retrying: %v", err) + time.Sleep(2 * time.Second) + } if err != nil { - log.Fatal(err) + return nil, fmt.Errorf("connect to postgres: %w", err) } - DB = db - - if err := DB.AutoMigrate(&models.Release{}); err != nil { - log.Fatal(err) + sqlDB, err := db.DB() + if err != nil { + return nil, err } + sqlDB.SetMaxOpenConns(10) + sqlDB.SetMaxIdleConns(5) + sqlDB.SetConnMaxLifetime(30 * time.Minute) + + if err := Migrate(db); err != nil { + return nil, err + } + + return db, nil +} + +// Migrate creates or updates the tables for all models. +func Migrate(db *gorm.DB) error { + if err := db.AutoMigrate(&models.Release{}); err != nil { + return fmt.Errorf("migrate: %w", err) + } + return nil +} + +func ping(db *gorm.DB) error { + sqlDB, err := db.DB() + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return sqlDB.PingContext(ctx) } diff --git a/internal/db/import.go b/internal/db/import.go new file mode 100644 index 0000000..eab4520 --- /dev/null +++ b/internal/db/import.go @@ -0,0 +1,75 @@ +package db + +import ( + "errors" + "fmt" + "io/fs" + "kooltap-updateserver/internal/models" + "log" + "os" + + "github.com/glebarez/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +// ImportSQLite copies releases from the old SQLite database at path into db, +// once. It does nothing when the file does not exist. After a successful +// import the file is renamed to path+".imported" so it is not imported again. +func ImportSQLite(db *gorm.DB, path string) error { + + if path == "" { + return nil + } + if _, err := os.Stat(path); errors.Is(err, fs.ErrNotExist) { + return nil + } else if err != nil { + return err + } + + var existing int64 + if err := db.Model(&models.Release{}).Count(&existing).Error; err != nil { + return err + } + if existing > 0 { + log.Printf("skipping SQLite import from %s: database already has %d releases", path, existing) + return nil + } + + src, err := gorm.Open(sqlite.Open(path), &gorm.Config{Logger: logger.Discard}) + if err != nil { + return fmt.Errorf("open sqlite %s: %w", path, err) + } + var releases []models.Release + err = src.Order("id").Find(&releases).Error + // Close before renaming the file below + if sqlDB, cerr := src.DB(); cerr == nil { + sqlDB.Close() + } + if err != nil { + return fmt.Errorf("read sqlite releases: %w", err) + } + + err = db.Transaction(func(tx *gorm.DB) error { + if len(releases) > 0 { + if err := tx.Create(&releases).Error; err != nil { + return err + } + } + // Rows were inserted with their original IDs, so move the sequence past them + if tx.Dialector.Name() == "postgres" { + return tx.Exec(`SELECT setval(pg_get_serial_sequence('releases', 'id'), COALESCE((SELECT MAX(id) FROM releases), 0) + 1, false)`).Error + } + return nil + }) + if err != nil { + return fmt.Errorf("import releases: %w", err) + } + + if err := os.Rename(path, path+".imported"); err != nil { + log.Printf("imported SQLite database but could not rename it: %v", err) + } + + log.Printf("imported %d releases from %s", len(releases), path) + return nil +} diff --git a/internal/db/import_test.go b/internal/db/import_test.go new file mode 100644 index 0000000..1a1936d --- /dev/null +++ b/internal/db/import_test.go @@ -0,0 +1,88 @@ +package db + +import ( + "errors" + "io/fs" + "kooltap-updateserver/internal/models" + "os" + "path/filepath" + "testing" + + "github.com/glebarez/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +func openSQLite(t *testing.T, dsn string) *gorm.DB { + t.Helper() + d, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{Logger: logger.Discard}) + if err != nil { + t.Fatal(err) + } + if err := Migrate(d); err != nil { + t.Fatal(err) + } + return d +} + +// target returns the database to import into: Postgres when TEST_DATABASE_URL +// is set, otherwise an in-memory SQLite database. +func target(t *testing.T) *gorm.DB { + t.Helper() + if dsn := os.Getenv("TEST_DATABASE_URL"); dsn != "" { + d, err := Connect(dsn) + if err != nil { + t.Fatal(err) + } + if err := d.Exec("TRUNCATE releases RESTART IDENTITY").Error; err != nil { + t.Fatal(err) + } + return d + } + return openSQLite(t, "file:"+t.Name()+"?mode=memory&cache=shared") +} + +func TestImportSQLite(t *testing.T) { + path := filepath.Join(t.TempDir(), "updater.db") + + old := openSQLite(t, path) + old.Create(&models.Release{Version: "1.0.0", VersionCode: 1, APKFile: "app-1.0.0.apk"}) + old.Create(&models.Release{Version: "1.1.0", VersionCode: 2, APKFile: "app-1.1.0.apk", Notes: "fixes"}) + old.Delete(&models.Release{}, 1) // leave a gap so IDs must be preserved, not renumbered + old.Create(&models.Release{Version: "1.2.0", VersionCode: 3, APKFile: "app-1.2.0.apk", Mandatory: true}) + if sqlDB, err := old.DB(); err == nil { + sqlDB.Close() + } + + dst := target(t) + if err := ImportSQLite(dst, path); err != nil { + t.Fatal(err) + } + + var got []models.Release + dst.Order("id").Find(&got) + if len(got) != 2 || got[0].ID != 2 || got[1].ID != 3 || got[0].Notes != "fixes" || !got[1].Mandatory { + t.Fatalf("imported %+v", got) + } + + if _, err := os.Stat(path); !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("sqlite file not renamed: %v", err) + } + if _, err := os.Stat(path + ".imported"); err != nil { + t.Fatal(err) + } + + // New rows must not collide with the imported IDs + next := models.Release{Version: "1.3.0", VersionCode: 4, APKFile: "app-1.3.0.apk"} + if err := dst.Create(&next).Error; err != nil { + t.Fatal(err) + } + if next.ID <= 3 { + t.Fatalf("new id %d collides with imported rows", next.ID) + } + + // A missing file is not an error + if err := ImportSQLite(dst, path); err != nil { + t.Fatal(err) + } +} diff --git a/internal/models/release.go b/internal/models/release.go index 30988ac..eca56d0 100644 --- a/internal/models/release.go +++ b/internal/models/release.go @@ -4,8 +4,8 @@ import "time" type Release struct { ID uint `gorm:"primaryKey"` - Version string `gorm:"unique;not null"` // versionName - VersionCode int64 `gorm:"not null"` // Android versionCode + Version string `gorm:"uniqueIndex;not null"` // versionName + VersionCode int64 `gorm:"uniqueIndex;not null"` // Android versionCode APKFile string `gorm:"not null"` Notes string diff --git a/internal/routes/files.go b/internal/routes/files.go new file mode 100644 index 0000000..593e8d5 --- /dev/null +++ b/internal/routes/files.go @@ -0,0 +1,74 @@ +package routes + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "io/fs" + "mime/multipart" + "os" +) + +// saveTemp writes an uploaded file to a new temp file in dir and returns its +// path and SHA-256. +func saveTemp(fh *multipart.FileHeader, dir string) (path, sum string, err error) { + + src, err := fh.Open() + if err != nil { + return "", "", err + } + defer src.Close() + + dst, err := os.CreateTemp(dir, "upload-*.apk") + if err != nil { + return "", "", err + } + + hash := sha256.New() + _, err = io.Copy(io.MultiWriter(dst, hash), src) + if cerr := dst.Close(); err == nil { + err = cerr + } + if err != nil { + os.Remove(dst.Name()) + return "", "", err + } + + return dst.Name(), hex.EncodeToString(hash.Sum(nil)), nil +} + +// moveFile moves src to dst without ever overwriting an existing dst. It +// falls back to a copy when they are on different filesystems (the tmp and +// uploads directories are separate volumes). +func moveFile(src, dst string) error { + + // Unlike rename, link fails when dst exists, so a concurrent upload can never replace a published APK + if err := os.Link(src, dst); err == nil { + return os.Remove(src) + } else if errors.Is(err, fs.ErrExist) { + return err + } + + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + + out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644) + if err != nil { + return err + } + + _, err = io.Copy(out, in) + if cerr := out.Close(); err == nil { + err = cerr + } + if err != nil { + os.Remove(dst) + return err + } + + return os.Remove(src) +} diff --git a/internal/routes/releases.go b/internal/routes/releases.go index b24004b..42e0326 100644 --- a/internal/routes/releases.go +++ b/internal/routes/releases.go @@ -1,79 +1,74 @@ package routes import ( - "crypto/sha256" - "encoding/hex" + "errors" "fmt" - "io" + "io/fs" "kooltap-updateserver/internal/apk_util" "kooltap-updateserver/internal/auth" - "kooltap-updateserver/internal/db" "kooltap-updateserver/internal/models" + "log" "net/http" "os" "path/filepath" + "regexp" + "strconv" "github.com/Masterminds/semver/v3" "github.com/gin-gonic/gin" + "gorm.io/gorm" ) -func Register(r *gin.Engine) { +// Handler serves the release API. +type Handler struct { + DB *gorm.DB + UploadDir string // published APKs, served under /downloads + TmpDir string // incoming uploads while they are inspected + MaxUpload int64 // maximum request body size in bytes for uploads +} + +func (h *Handler) Register(r *gin.Engine) { api := r.Group("/api") { - api.GET("/update", CheckUpdate) - api.GET("/releases", ListReleases) + api.GET("/update", h.CheckUpdate) + api.GET("/releases", h.ListReleases) } - r.GET("/download/latest", DownloadLatest) + r.GET("/download/latest", h.DownloadLatest) admin := r.Group("/admin") admin.Use(auth.AdminAuth()) { - admin.POST("/releases", UploadRelease) - admin.DELETE("/releases/:id", DeleteRelease) + admin.POST("/releases", h.UploadRelease) + admin.DELETE("/releases/:id", h.DeleteRelease) } } -func CheckUpdate(c *gin.Context) { +// CheckUpdate tells a tablet whether a newer release exists. Tablets send their +// versionName as "version" and, optionally, their versionCode as "code"; the +// code is the most reliable comparison, so it wins when present. +func (h *Handler) CheckUpdate(c *gin.Context) { - current := c.Query("version") - - var latest models.Release - - // The highest Android version code is the latest build, regardless of upload order - if err := latestRelease(&latest); err != nil { - - c.JSON(http.StatusOK, gin.H{ - "update": false, - }) + latest, err := h.latestRelease() + if errors.Is(err, gorm.ErrRecordNotFound) { + c.JSON(http.StatusOK, gin.H{"update": false}) + return + } + if err != nil { + internalError(c, err) return } - cv, err1 := semver.NewVersion(current) - lv, err2 := semver.NewVersion(latest.Version) - - if err1 != nil || err2 != nil { - - c.JSON(http.StatusOK, gin.H{ - "update": latest.Version != current, - }) - - return - } - - if !lv.GreaterThan(cv) { - - c.JSON(http.StatusOK, gin.H{ - "update": false, - }) - + if !isNewer(latest, c.Query("version"), c.Query("code")) { + c.JSON(http.StatusOK, gin.H{"update": false}) return } c.JSON(http.StatusOK, gin.H{ "update": true, "version": latest.Version, + "code": latest.VersionCode, "notes": latest.Notes, "mandatory": latest.Mandatory, "sha256": latest.SHA256, @@ -81,75 +76,105 @@ func CheckUpdate(c *gin.Context) { }) } +func isNewer(latest *models.Release, version, code string) bool { + + if code != "" { + if cc, err := strconv.ParseInt(code, 10, 64); err == nil { + return latest.VersionCode > cc + } + } + + cv, err1 := semver.NewVersion(version) + lv, err2 := semver.NewVersion(latest.Version) + if err1 != nil || err2 != nil { + return latest.Version != version + } + + return lv.GreaterThan(cv) +} + // DownloadLatest redirects to the newest APK, for installing on a new tablet from a browser -func DownloadLatest(c *gin.Context) { +func (h *Handler) DownloadLatest(c *gin.Context) { - var latest models.Release - - if err := latestRelease(&latest); err != nil { - c.JSON(http.StatusNotFound, gin.H{ - "error": "No releases yet", - }) + latest, err := h.latestRelease() + if errors.Is(err, gorm.ErrRecordNotFound) { + c.JSON(http.StatusNotFound, gin.H{"error": "No releases yet"}) + return + } + if err != nil { + internalError(c, err) return } c.Redirect(http.StatusFound, "/downloads/"+latest.APKFile) } -func latestRelease(release *models.Release) error { - return db.DB. - Order("version_code desc"). - First(release).Error +// latestRelease returns the release with the highest Android version code, +// which is the latest build regardless of upload order. +func (h *Handler) latestRelease() (*models.Release, error) { + var release models.Release + err := h.DB.Order("version_code desc").First(&release).Error + if err != nil { + return nil, err + } + return &release, nil } -func ListReleases(c *gin.Context) { +func (h *Handler) ListReleases(c *gin.Context) { - var releases []models.Release + releases := []models.Release{} - db.DB. - Order("created_at desc"). - Find(&releases) + if err := h.DB.Order("created_at desc").Find(&releases).Error; err != nil { + internalError(c, err) + return + } c.JSON(http.StatusOK, releases) } -func UploadRelease(c *gin.Context) { +func (h *Handler) UploadRelease(c *gin.Context) { + + if h.MaxUpload > 0 { + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, h.MaxUpload) + } notes := c.PostForm("notes") mandatory := c.PostForm("mandatory") == "true" file, err := c.FormFile("apk") if err != nil { - c.JSON(http.StatusBadRequest, gin.H{ - "error": "APK required", - }) + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + c.JSON(http.StatusRequestEntityTooLarge, gin.H{"error": "APK too large"}) + return + } + c.JSON(http.StatusBadRequest, gin.H{"error": "APK required"}) return } - // Save temporarily first because we need to inspect the APK - tempPath := filepath.Join("uploads", "temp-"+file.Filename) - - if err := c.SaveUploadedFile(file, tempPath); err != nil { - c.JSON(http.StatusInternalServerError, gin.H{ - "error": err.Error(), - }) + // Save to a private temp file first because we need to inspect the APK; + // the hash is computed while copying so the file is only read once. + tempPath, sum, err := saveTemp(file, h.TmpDir) + if err != nil { + internalError(c, err) return } - defer os.Remove(tempPath) // Extract version information from APK info, err := apk_util.Read(tempPath) if err != nil { - c.JSON(http.StatusBadRequest, gin.H{ - "error": "invalid APK: " + err.Error(), - }) + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid APK: " + err.Error()}) return } // Tablets only see the release with the highest version code, so an older or equal one would never be offered - var latest models.Release - if err := latestRelease(&latest); err == nil && int64(info.VersionCode) <= latest.VersionCode { + latest, err := h.latestRelease() + if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + internalError(c, err) + return + } + if latest != nil && int64(info.VersionCode) <= latest.VersionCode { c.JSON(http.StatusConflict, gin.H{ "error": fmt.Sprintf("version code %d must be higher than the latest release (%s, code %d)", info.VersionCode, latest.Version, latest.VersionCode), @@ -157,42 +182,22 @@ func UploadRelease(c *gin.Context) { return } - filename := "app-" + info.Version + ".apk" - path := filepath.Join("uploads", filename) + // versionName comes from the uploaded file, so keep it from escaping the upload directory + filename := "app-" + safeName(info.Version) + ".apk" + path := filepath.Join(h.UploadDir, filename) // Prevent overwriting an existing release if _, err := os.Stat(path); err == nil { - c.JSON(http.StatusConflict, gin.H{ - "error": "version already exists", - }) + c.JSON(http.StatusConflict, gin.H{"error": "version already exists"}) return } - // Move temp APK to final filename - if err := os.Rename(tempPath, path); err != nil { - c.JSON(http.StatusInternalServerError, gin.H{ - "error": err.Error(), - }) - return - } - - // Calculate SHA256 - hash := sha256.New() - - f, err := os.Open(path) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{ - "error": err.Error(), - }) - return - } - - defer f.Close() - - if _, err := io.Copy(hash, f); err != nil { - c.JSON(http.StatusInternalServerError, gin.H{ - "error": err.Error(), - }) + if err := moveFile(tempPath, path); err != nil { + if errors.Is(err, fs.ErrExist) { + c.JSON(http.StatusConflict, gin.H{"error": "version already exists"}) + return + } + internalError(c, err) return } @@ -202,41 +207,70 @@ func UploadRelease(c *gin.Context) { APKFile: filename, Notes: notes, Mandatory: mandatory, - SHA256: hex.EncodeToString(hash.Sum(nil)), + SHA256: sum, } - if err := db.DB.Create(&release).Error; err != nil { + if err := h.DB.Create(&release).Error; err != nil { // cleanup uploaded file if DB fails os.Remove(path) - c.JSON(http.StatusInternalServerError, gin.H{ - "error": err.Error(), - }) + // A concurrent upload of the same version won the race + if errors.Is(err, gorm.ErrDuplicatedKey) { + c.JSON(http.StatusConflict, gin.H{"error": "version already exists"}) + return + } + + internalError(c, err) return } c.JSON(http.StatusOK, release) } -func DeleteRelease(c *gin.Context) { - - var release models.Release - - if err := db.DB.First(&release, c.Param("id")).Error; err != nil { - - c.JSON(http.StatusNotFound, gin.H{ - "error": "Not found", - }) +func (h *Handler) DeleteRelease(c *gin.Context) { + id, err := strconv.ParseUint(c.Param("id"), 10, 64) + if err != nil { + c.JSON(http.StatusNotFound, gin.H{"error": "Not found"}) return } - os.Remove(filepath.Join("uploads", release.APKFile)) + var release models.Release - db.DB.Delete(&release) + if err := h.DB.First(&release, id).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + c.JSON(http.StatusNotFound, gin.H{"error": "Not found"}) + return + } + internalError(c, err) + return + } - c.JSON(http.StatusOK, gin.H{ - "success": true, - }) + // Remove the row first: a leftover file is harmless, a row pointing at a missing file is not + if err := h.DB.Delete(&release).Error; err != nil { + internalError(c, err) + return + } + + if err := os.Remove(filepath.Join(h.UploadDir, release.APKFile)); err != nil && !errors.Is(err, os.ErrNotExist) { + log.Printf("release %d deleted but removing %s failed: %v", release.ID, release.APKFile, err) + } + + c.JSON(http.StatusOK, gin.H{"success": true}) +} + +func internalError(c *gin.Context, err error) { + log.Printf("%s %s: %v", c.Request.Method, c.Request.URL.Path, err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "internal server error"}) +} + +var unsafeChars = regexp.MustCompile(`[^A-Za-z0-9._+-]`) + +func safeName(s string) string { + s = unsafeChars.ReplaceAllString(s, "_") + if s == "" || s == "." || s == ".." { + s = "_" + } + return s } diff --git a/internal/routes/releases_test.go b/internal/routes/releases_test.go new file mode 100644 index 0000000..8809222 --- /dev/null +++ b/internal/routes/releases_test.go @@ -0,0 +1,171 @@ +package routes + +import ( + "encoding/json" + "errors" + "io/fs" + "kooltap-updateserver/internal/db" + "kooltap-updateserver/internal/models" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/gin-gonic/gin" + "github.com/glebarez/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +func newHandler(t *testing.T, releases ...models.Release) (*Handler, *gin.Engine) { + t.Helper() + gin.SetMode(gin.TestMode) + + database, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{ + TranslateError: true, + Logger: logger.Discard, + }) + if err != nil { + t.Fatal(err) + } + if err := db.Migrate(database); err != nil { + t.Fatal(err) + } + for i := range releases { + if err := database.Create(&releases[i]).Error; err != nil { + t.Fatal(err) + } + } + + h := &Handler{DB: database, UploadDir: t.TempDir(), TmpDir: t.TempDir()} + r := gin.New() + h.Register(r) + return h, r +} + +func do(r *gin.Engine, method, url string) (*httptest.ResponseRecorder, map[string]any) { + req := httptest.NewRequest(method, url, nil) + req.Header.Set("X-Admin-Token", "secret") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + var body map[string]any + _ = json.Unmarshal(w.Body.Bytes(), &body) + return w, body +} + +func TestCheckUpdate(t *testing.T) { + _, r := newHandler(t, + models.Release{Version: "1.2.0", VersionCode: 12, APKFile: "app-1.2.0.apk"}, + models.Release{Version: "1.10.0", VersionCode: 110, APKFile: "app-1.10.0.apk", Mandatory: true}, + ) + + cases := []struct { + query string + want bool + }{ + {"version=1.2.0", true}, + {"version=1.10.0", false}, + {"version=2.0.0", false}, + {"version=1.2.0&code=12", true}, + {"version=1.10.0&code=110", false}, + {"version=garbage", true}, + {"version=garbage&code=200", false}, + } + for _, tc := range cases { + t.Run(tc.query, func(t *testing.T) { + w, body := do(r, http.MethodGet, "/api/update?"+tc.query) + if w.Code != http.StatusOK { + t.Fatalf("status %d", w.Code) + } + if body["update"] != tc.want { + t.Fatalf("update = %v, want %v", body["update"], tc.want) + } + if tc.want && body["download"] != "/downloads/app-1.10.0.apk" { + t.Fatalf("download = %v", body["download"]) + } + }) + } +} + +func TestCheckUpdateNoReleases(t *testing.T) { + _, r := newHandler(t) + _, body := do(r, http.MethodGet, "/api/update?version=1.0.0") + if body["update"] != false { + t.Fatalf("update = %v", body["update"]) + } + if w, _ := do(r, http.MethodGet, "/download/latest"); w.Code != http.StatusNotFound { + t.Fatalf("download latest status %d", w.Code) + } +} + +func TestListReleasesEmptyIsArray(t *testing.T) { + _, r := newHandler(t) + w, _ := do(r, http.MethodGet, "/api/releases") + if w.Body.String() != "[]" { + t.Fatalf("body = %s", w.Body.String()) + } +} + +func TestDeleteRelease(t *testing.T) { + t.Setenv("ADMIN_TOKEN", "secret") + h, r := newHandler(t, models.Release{Version: "1.0.0", VersionCode: 1, APKFile: "app-1.0.0.apk"}) + + apk := filepath.Join(h.UploadDir, "app-1.0.0.apk") + if err := os.WriteFile(apk, []byte("x"), 0644); err != nil { + t.Fatal(err) + } + + if w, _ := do(r, http.MethodDelete, "/admin/releases/abc"); w.Code != http.StatusNotFound { + t.Fatalf("bad id status %d", w.Code) + } + if w, _ := do(r, http.MethodDelete, "/admin/releases/1"); w.Code != http.StatusOK { + t.Fatalf("delete status %d", w.Code) + } + if _, err := os.Stat(apk); !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("apk still exists: %v", err) + } + if w, _ := do(r, http.MethodDelete, "/admin/releases/1"); w.Code != http.StatusNotFound { + t.Fatalf("second delete status %d", w.Code) + } +} + +func TestMoveFileNeverOverwrites(t *testing.T) { + dir := t.TempDir() + src := filepath.Join(dir, "src") + dst := filepath.Join(dir, "dst") + os.WriteFile(src, []byte("new"), 0644) + os.WriteFile(dst, []byte("old"), 0644) + + if err := moveFile(src, dst); !errors.Is(err, fs.ErrExist) { + t.Fatalf("err = %v, want ErrExist", err) + } + if b, _ := os.ReadFile(dst); string(b) != "old" { + t.Fatalf("dst overwritten: %q", b) + } + + os.Remove(dst) + if err := moveFile(src, dst); err != nil { + t.Fatal(err) + } + if b, _ := os.ReadFile(dst); string(b) != "new" { + t.Fatalf("dst = %q", b) + } +} + +func TestSafeName(t *testing.T) { + cases := map[string]string{ + "1.2.3": "1.2.3", + "1.0.0-beta+7": "1.0.0-beta+7", + "../../etc/x": ".._.._etc_x", + "..": "_", + "": "_", + `1.0\..\evil`: "1.0_.._evil", + "1.0 (release)": "1.0__release_", + } + for in, want := range cases { + if got := safeName(in); got != want { + t.Errorf("safeName(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/internal/updater.go b/internal/updater.go deleted file mode 100644 index 5bf0569..0000000 --- a/internal/updater.go +++ /dev/null @@ -1 +0,0 @@ -package internal