test(endtoend): Use Docker to start database servers

Remove setup-mysql and setup-postgresql actions

Author: kyleconroy

.github/workflows/ci.yml

@@ -53,24 +53,24 @@ jobs:
         CGO_ENABLED: ${{ matrix.cgo }}
 
     # Start a PostgreSQL server
-    - uses: sqlc-dev/action-setup-postgres@master
-      with:
-        postgres-version: "16"
-      id: postgres
+    # - uses: sqlc-dev/action-setup-postgres@master
+    #   with:
+    #     postgres-version: "16"
+    #   id: postgres
 
     # Start a MySQL server
-    - uses: shogo82148/actions-setup-mysql@v1
-      with:
-        mysql-version: "9.0"
+    # - uses: shogo82148/actions-setup-mysql@v1
+    #   with:
+    #     mysql-version: "9.0"
 
     - name: test ./...
       run: gotestsum --junitfile junit.xml -- --tags=examples -timeout 20m ./...
       env:
         CI_SQLC_PROJECT_ID: ${{ secrets.CI_SQLC_PROJECT_ID }}
         CI_SQLC_AUTH_TOKEN: ${{ secrets.CI_SQLC_AUTH_TOKEN }}
         SQLC_AUTH_TOKEN: ${{ secrets.CI_SQLC_AUTH_TOKEN }}
-        MYSQL_SERVER_URI: root:@tcp(localhost:3306)/mysql?multiStatements=true&parseTime=true
-        POSTGRESQL_SERVER_URI: ${{ steps.postgres.outputs.connection-uri }}?sslmode=disable
+        # MYSQL_SERVER_URI: root:@tcp(localhost:3306)/mysql?multiStatements=true&parseTime=true
+        # POSTGRESQL_SERVER_URI: ${{ steps.postgres.outputs.connection-uri }}?sslmode=disable
         CGO_ENABLED: ${{ matrix.cgo }}
 
   vuln_check:

internal/endtoend/endtoend_test.go

@@ -17,7 +17,7 @@ import (
 	"github.com/sqlc-dev/sqlc/internal/cmd"
 	"github.com/sqlc-dev/sqlc/internal/config"
 	"github.com/sqlc-dev/sqlc/internal/opts"
-	"github.com/sqlc-dev/sqlc/internal/sqltest/local"
+	"github.com/sqlc-dev/sqlc/internal/sqltest/docker"
 )
 
 func lineEndings() cmp.Option {
@@ -112,6 +112,24 @@ func TestReplay(t *testing.T) {
 	// t.Parallel()
 	ctx := context.Background()
 
+	var mysqlURI, postgresURI string
+	if err := docker.Installed(); err == nil {
+		{
+			host, err := docker.StartPostgreSQLServer(ctx)
+			if err != nil {
+				t.Fatalf("starting postgresql failed: %s", err)
+			}
+			postgresURI = host
+		}
+		{
+			host, err := docker.StartMySQLServer(ctx)
+			if err != nil {
+				t.Fatalf("starting mysql failed: %s", err)
+			}
+			mysqlURI = host
+		}
+	}
+
 	contexts := map[string]textContext{
 		"base": {
 			Mutate:  func(t *testing.T, path string) func(*config.Config) { return func(c *config.Config) {} },
@@ -124,13 +142,13 @@ func TestReplay(t *testing.T) {
 						{
 							Name:   "postgres",
 							Engine: config.EnginePostgreSQL,
-							URI:    local.PostgreSQLServer(),
+							URI:    postgresURI,
 						},
 
 						{
 							Name:   "mysql",
 							Engine: config.EngineMySQL,
-							URI:    local.MySQLServer(),
+							URI:    mysqlURI,
 						},
 					}
 					for i := range c.SQL {
@@ -150,13 +168,8 @@ func TestReplay(t *testing.T) {
 				}
 			},
 			Enabled: func() bool {
-				if len(os.Getenv("POSTGRESQL_SERVER_URI")) == 0 {
-					return false
-				}
-				if len(os.Getenv("MYSQL_SERVER_URI")) == 0 {
-					return false
-				}
-				return true
+				err := docker.Installed()
+				return err == nil
 			},
 		},
 	}

internal/sqltest/docker/enabled.go

@@ -0,0 +1,13 @@
+package docker
+
+import (
+	"fmt"
+	"os/exec"
+)
+
+func Installed() error {
+	if _, err := exec.LookPath("docker"); err != nil {
+		return fmt.Errorf("docker not found: %w", err)
+	}
+	return nil
+}

internal/sqltest/docker/mysql.go

@@ -0,0 +1,60 @@
+package docker
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"os/exec"
+	"time"
+
+	_ "github.com/go-sql-driver/mysql"
+)
+
+func StartMySQLServer(c context.Context) (string, error) {
+	if err := Installed(); err != nil {
+		return "", err
+	}
+
+	ctx, cancel := context.WithTimeout(c, 10*time.Second)
+	defer cancel()
+
+	cmd := exec.Command("docker", "run",
+		"--name", "sqlc_sqltest_docker_mysql",
+		"-e", "MYSQL_ROOT_PASSWORD=mysecretpassword",
+		"-e", "MYSQL_DATABASE=dinotest",
+		"-p", "3306:3306",
+		"-d",
+		"mysql:8",
+	)
+
+	output, err := cmd.CombinedOutput()
+	fmt.Println(string(output))
+	if err != nil {
+		return "", err
+	}
+
+	// Create a ticker that fires every 10ms
+	ticker := time.NewTicker(10 * time.Millisecond)
+	defer ticker.Stop()
+
+	uri := "root:mysecretpassword@/dinotest"
+
+	db, err := sql.Open("mysql", uri)
+	if err != nil {
+		return "", fmt.Errorf("sql.Open: %w", err)
+	}
+
+	for {
+		select {
+		case <-ctx.Done():
+			return "", fmt.Errorf("timeout reached: %w", ctx.Err())
+
+		case <-ticker.C:
+			// Run your function here
+			if err := db.PingContext(ctx); err != nil {
+				continue
+			}
+			return uri, nil
+		}
+	}
+}

internal/sqltest/docker/postgres.go

@@ -0,0 +1,62 @@
+package docker
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"os/exec"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+)
+
+func StartPostgreSQLServer(c context.Context) (string, error) {
+	if err := Installed(); err != nil {
+		return "", err
+	}
+
+	ctx, cancel := context.WithTimeout(c, 5*time.Second)
+	defer cancel()
+
+	cmd := exec.Command("docker", "run",
+		"--name", "sqlc_sqltest_docker_postgres",
+		"-e", "POSTGRES_PASSWORD=mysecretpassword",
+		"-e", "POSTGRES_USER=postgres",
+		"-p", "5432:5432",
+		"-d",
+		"postgres:16",
+		"-c", "max_connections=200",
+	)
+
+	output, err := cmd.CombinedOutput()
+	fmt.Println(string(output))
+	if err != nil {
+		return "", err
+	}
+
+	// Create a ticker that fires every 10ms
+	ticker := time.NewTicker(10 * time.Millisecond)
+	defer ticker.Stop()
+
+	uri := "postgres://postgres:mysecretpassword@localhost:5432/postgres?sslmode=disable"
+
+	for {
+		select {
+		case <-ctx.Done():
+			return "", fmt.Errorf("timeout reached: %w", ctx.Err())
+
+		case <-ticker.C:
+			// Run your function here
+			conn, err := pgx.Connect(ctx, uri)
+			if err != nil {
+				slog.Debug("sqltest", "connect", err)
+				continue
+			}
+			if err := conn.Ping(ctx); err != nil {
+				slog.Error("sqltest", "ping", err)
+				continue
+			}
+			return uri, nil
+		}
+	}
+}