Added uncertainty circle and zipping

datapoint/batch.go

@@ -0,0 +1,80 @@
+package datapoint
+
+import (
+	"archive/zip"
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+)
+
+const (
+	MaxBatchBytes     = 4 << 20  // Maximum ZIP payload size on the wire.
+	MaxBatchJSONBytes = 16 << 20 // Maximum expanded JSON size.
+)
+
+// ZipBatch packages GPS fixes into an in-memory ZIP containing points.json.
+func ZipBatch(points []DataPoint) ([]byte, error) {
+	raw, err := json.Marshal(points)
+	if err != nil {
+		return nil, fmt.Errorf("encode points: %w", err)
+	}
+	if len(raw) > MaxBatchJSONBytes {
+		return nil, fmt.Errorf("batch JSON exceeds %d bytes", MaxBatchJSONBytes)
+	}
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	file, err := zw.Create("points.json")
+	if err != nil {
+		return nil, err
+	}
+	if _, err = file.Write(raw); err != nil {
+		return nil, err
+	}
+	if err = zw.Close(); err != nil {
+		return nil, err
+	}
+	if buf.Len() > MaxBatchBytes {
+		return nil, fmt.Errorf("ZIP batch exceeds %d bytes", MaxBatchBytes)
+	}
+	return buf.Bytes(), nil
+}
+
+// UnzipBatch reads points in memory; archive entries are never written to disk.
+// Legacy JSON batches are accepted so the server can be upgraded first.
+func UnzipBatch(payload []byte) ([]DataPoint, error) {
+	if len(payload) > MaxBatchBytes {
+		return nil, fmt.Errorf("batch exceeds %d bytes", MaxBatchBytes)
+	}
+	raw := payload
+	if !bytes.HasPrefix(bytes.TrimSpace(payload), []byte("[")) && !bytes.Equal(bytes.TrimSpace(payload), []byte("null")) {
+		zr, err := zip.NewReader(bytes.NewReader(payload), int64(len(payload)))
+		if err != nil {
+			return nil, fmt.Errorf("open ZIP batch: %w", err)
+		}
+		if len(zr.File) != 1 || zr.File[0].Name != "points.json" {
+			return nil, fmt.Errorf("ZIP batch must contain only points.json")
+		}
+		file := zr.File[0]
+		if file.UncompressedSize64 > MaxBatchJSONBytes {
+			return nil, fmt.Errorf("expanded batch exceeds %d bytes", MaxBatchJSONBytes)
+		}
+		r, err := file.Open()
+		if err != nil {
+			return nil, fmt.Errorf("open points.json: %w", err)
+		}
+		defer r.Close()
+		raw, err = io.ReadAll(io.LimitReader(r, MaxBatchJSONBytes+1))
+		if err != nil {
+			return nil, fmt.Errorf("read points.json: %w", err)
+		}
+		if len(raw) > MaxBatchJSONBytes {
+			return nil, fmt.Errorf("expanded batch exceeds %d bytes", MaxBatchJSONBytes)
+		}
+	}
+	var points []DataPoint
+	if err := json.Unmarshal(raw, &points); err != nil {
+		return nil, fmt.Errorf("decode points: %w", err)
+	}
+	return points, nil
+}

datapoint/batch_test.go

@@ -0,0 +1,103 @@
+package datapoint
+
+import (
+	"archive/zip"
+	"bytes"
+	"encoding/json"
+	"math"
+	"reflect"
+	"testing"
+)
+
+func TestBatchRoundTrip(t *testing.T) {
+	speed := 14.5
+	points := make([]DataPoint, 25)
+	for i := range points {
+		points[i] = DataPoint{Timestamp: uint64(1000 + i), Lat: 41.8, Long: -87.6, LatErr: 3, LongErr: 4, Speed: &speed}
+	}
+	raw, err := json.Marshal(points)
+	if err != nil {
+		t.Fatal(err)
+	}
+	zipped, err := ZipBatch(points)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(zipped) >= len(raw) {
+		t.Fatalf("batch did not compress: %d >= %d", len(zipped), len(raw))
+	}
+	for name, payload := range map[string][]byte{"zip": zipped, "legacy": raw} {
+		t.Run(name, func(t *testing.T) {
+			got, err := UnzipBatch(payload)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !reflect.DeepEqual(got, points) {
+				t.Fatalf("round trip changed points: %+v", got)
+			}
+		})
+	}
+	for _, points := range [][]DataPoint{nil, {}} {
+		payload, err := ZipBatch(points)
+		if err != nil {
+			t.Fatal(err)
+		}
+		got, err := UnzipBatch(payload)
+		if err != nil || !reflect.DeepEqual(got, points) {
+			t.Fatalf("empty batch round trip: %v, %v", got, err)
+		}
+	}
+}
+
+func archive(t *testing.T, names []string, raw []byte) []byte {
+	t.Helper()
+	var b bytes.Buffer
+	w := zip.NewWriter(&b)
+	for _, name := range names {
+		f, err := w.Create(name)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if _, err = f.Write(raw); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+	return b.Bytes()
+}
+
+func TestUnzipRejectsBadBatches(t *testing.T) {
+	valid := archive(t, []string{"points.json"}, []byte("[]"))
+	// Alter the stored CRC, leaving a structurally readable archive.
+	corrupt := bytes.Clone(valid)
+	central := bytes.Index(corrupt, []byte{'P', 'K', 1, 2})
+	if central < 0 {
+		t.Fatal("missing ZIP directory")
+	}
+	corrupt[central+16] ^= 0xff
+	cases := map[string][]byte{
+		"garbage":              []byte("not a zip"),
+		"truncated":            valid[:len(valid)-10],
+		"invalid json":         archive(t, []string{"points.json"}, []byte("bad JSON")),
+		"wrong entry":          archive(t, []string{"../points.json"}, []byte("[]")),
+		"multiple entries":     archive(t, []string{"points.json", "other.json"}, []byte("[]")),
+		"oversized compressed": make([]byte, MaxBatchBytes+1),
+		"oversized expanded":   archive(t, []string{"points.json"}, bytes.Repeat([]byte(" "), MaxBatchJSONBytes+1)),
+		"checksum":             corrupt,
+	}
+	for name, payload := range cases {
+		t.Run(name, func(t *testing.T) {
+			if _, err := UnzipBatch(payload); err == nil {
+				t.Fatal("accepted invalid batch")
+			}
+		})
+	}
+}
+
+func TestZipRejectsInvalidPoints(t *testing.T) {
+	if _, err := ZipBatch([]DataPoint{{Lat: math.NaN()}}); err == nil {
+		t.Fatal("accepted non-JSON data")
+	}
+}