Home Explore
Sign In
enzo
/
palmoil-mobile
1
0
Fork 0
Files Issues 0 Pull Requests 0 Wiki
Browse Source

simple documentation for centralized data collection

enzo 5 days ago
parent
e887bd837a
commit
7e3292e78f
2 changed files with 642 additions and 0 deletions
Unified View Show Diff Stats
  1. 537 0
      DATA_COLLECTION_PIPELINE.html
  2. 105 0
      DATA_COLLECTION_PIPELINE.md

+ 537 - 0
DATA_COLLECTION_PIPELINE.html
View File 

@@ -0,0 +1,537 @@
 
+<!doctype html>
 
+<html lang="en">
 
+<head>
 
+<meta charset="utf-8">
 
+<meta name="viewport" content="width=device-width, initial-scale=1">
 
+<title>Field-to-Model: Correction Data Pipeline</title>
 
+<style>
 
+  :root {
 
+    --bg: #F2F1E7;
 
+    --bg-alt: #E7E4D5;
 
+    --panel-line: rgba(27,36,30,0.16);
 
+    --ink: #1B241E;
 
+    --ink-dim: rgba(27,36,30,0.64);
 
+    --ink-faint: rgba(27,36,30,0.42);
 
+    --line: rgba(27,36,30,0.14);
 
+    --accent: #B9711F;
 
+    --accent-soft: rgba(185,113,31,0.12);
 
+    --accent-ink: #FBF7EE;
 
+    --green: #1F7A5C;
 
+    --green-soft: rgba(31,122,92,0.10);
 
+    --red: #A3402E;
 
+    --red-soft: rgba(163,64,46,0.10);
 
+    --mono: ui-monospace, "SF Mono", "Cascadia Code", "Roboto Mono", Consolas, "Liberation Mono", monospace;
 
+    --serif: Georgia, "Iowan Old Style", "Palatino Linotype", "Book Antiqua", serif;
 
+    --shadow: 0 1px 2px rgba(27,36,30,0.06), 0 8px 24px rgba(27,36,30,0.05);
 
+  }
 
+
 
+  @media (prefers-color-scheme: dark) {
 
+    :root {
 
+      --bg: #101B16;
 
+      --bg-alt: #17251E;
 
+      --panel-line: rgba(233,231,220,0.14);
 
+      --ink: #E9E7DC;
 
+      --ink-dim: rgba(233,231,220,0.66);
 
+      --ink-faint: rgba(233,231,220,0.42);
 
+      --line: rgba(233,231,220,0.13);
 
+      --accent: #DDA35C;
 
+      --accent-soft: rgba(221,163,92,0.14);
 
+      --accent-ink: #17130C;
 
+      --green: #55C29A;
 
+      --green-soft: rgba(85,194,154,0.12);
 
+      --red: #D97F6C;
 
+      --red-soft: rgba(217,127,108,0.12);
 
+      --shadow: 0 1px 2px rgba(0,0,0,0.3), 0 8px 28px rgba(0,0,0,0.28);
 
+    }
 
+  }
 
+  :root[data-theme="dark"] {
 
+    --bg: #101B16; --bg-alt: #17251E; --panel-line: rgba(233,231,220,0.14);
 
+    --ink: #E9E7DC; --ink-dim: rgba(233,231,220,0.66); --ink-faint: rgba(233,231,220,0.42);
 
+    --line: rgba(233,231,220,0.13); --accent: #DDA35C; --accent-soft: rgba(221,163,92,0.14);
 
+    --accent-ink: #17130C; --green: #55C29A; --green-soft: rgba(85,194,154,0.12);
 
+    --red: #D97F6C; --red-soft: rgba(217,127,108,0.12);
 
+    --shadow: 0 1px 2px rgba(0,0,0,0.3), 0 8px 28px rgba(0,0,0,0.28);
 
+  }
 
+  :root[data-theme="light"] {
 
+    --bg: #F2F1E7; --bg-alt: #E7E4D5; --panel-line: rgba(27,36,30,0.16);
 
+    --ink: #1B241E; --ink-dim: rgba(27,36,30,0.64); --ink-faint: rgba(27,36,30,0.42);
 
+    --line: rgba(27,36,30,0.14); --accent: #B9711F; --accent-soft: rgba(185,113,31,0.12);
 
+    --accent-ink: #FBF7EE; --green: #1F7A5C; --green-soft: rgba(31,122,92,0.10);
 
+    --red: #A3402E; --red-soft: rgba(163,64,46,0.10);
 
+    --shadow: 0 1px 2px rgba(27,36,30,0.06), 0 8px 24px rgba(27,36,30,0.05);
 
+  }
 
+
 
+  * { box-sizing: border-box; }
 
+  html, body { margin: 0; padding: 0; }
 
+  body {
 
+    background: var(--bg);
 
+    color: var(--ink);
 
+    font-family: var(--serif);
 
+    font-size: 17px;
 
+    line-height: 1.65;
 
+    -webkit-font-smoothing: antialiased;
 
+  }
 
+  a { color: var(--accent); }
 
+  a:focus-visible, button:focus-visible { outline: 2px solid var(--accent); outline-offset: 3px; }
 
+
 
+  .wrap { max-width: 800px; margin: 0 auto; padding: 56px 24px 96px; }
 
+
 
+  .eyebrow {
 
+    font-family: var(--mono);
 
+    font-size: 12px;
 
+    letter-spacing: 0.11em;
 
+    text-transform: uppercase;
 
+    color: var(--accent);
 
+    display: inline-flex;
 
+    align-items: center;
 
+    gap: 8px;
 
+    margin-bottom: 18px;
 
+  }
 
+  .eyebrow::before {
 
+    content: "";
 
+    width: 7px; height: 7px;
 
+    background: var(--accent);
 
+    border-radius: 50%;
 
+    display: inline-block;
 
+  }
 
+
 
+  h1 {
 
+    font-family: var(--mono);
 
+    font-weight: 600;
 
+    font-size: clamp(28px, 4.4vw, 38px);
 
+    line-height: 1.18;
 
+    letter-spacing: -0.01em;
 
+    text-wrap: balance;
 
+    margin: 0 0 18px;
 
+    color: var(--ink);
 
+  }
 
+  .subtitle {
 
+    font-family: var(--serif);
 
+    font-style: italic;
 
+    font-size: 19px;
 
+    color: var(--ink-dim);
 
+    max-width: 62ch;
 
+    text-wrap: balance;
 
+    margin: 0 0 8px;
 
+  }
 
+  .meta-row {
 
+    display: flex;
 
+    flex-wrap: wrap;
 
+    gap: 18px;
 
+    margin-top: 28px;
 
+    padding-top: 20px;
 
+    border-top: 1px solid var(--line);
 
+    font-family: var(--mono);
 
+    font-size: 12px;
 
+    color: var(--ink-faint);
 
+  }
 
+  .meta-row span b { color: var(--ink-dim); font-weight: 600; }
 
+
 
+  nav.toc {
 
+    margin-top: 32px;
 
+    padding: 18px 20px;
 
+    border: 1px solid var(--line);
 
+    border-radius: 10px;
 
+    background: var(--bg-alt);
 
+  }
 
+  nav.toc p {
 
+    font-family: var(--mono);
 
+    font-size: 11px;
 
+    letter-spacing: 0.09em;
 
+    text-transform: uppercase;
 
+    color: var(--ink-faint);
 
+    margin: 0 0 10px;
 
+  }
 
+  nav.toc ol {
 
+    margin: 0; padding: 0; list-style: none;
 
+    columns: 2;
 
+    column-gap: 24px;
 
+    font-family: var(--mono);
 
+    font-size: 13px;
 
+  }
 
+  nav.toc li { break-inside: avoid; margin-bottom: 8px; }
 
+  nav.toc a { text-decoration: none; color: var(--ink-dim); }
 
+  nav.toc a:hover { color: var(--accent); }
 
+  nav.toc a .n { color: var(--ink-faint); margin-right: 6px; }
 
+
 
+  section { margin-top: 64px; scroll-margin-top: 24px; }
 
+  .sec-head {
 
+    display: flex;
 
+    align-items: baseline;
 
+    gap: 12px;
 
+    border-bottom: 1px solid var(--line);
 
+    padding-bottom: 10px;
 
+    margin-bottom: 22px;
 
+  }
 
+  .sec-head .n {
 
+    font-family: var(--mono);
 
+    font-size: 13px;
 
+    color: var(--accent);
 
+    letter-spacing: 0.04em;
 
+  }
 
+  .sec-head h2 {
 
+    font-family: var(--mono);
 
+    font-size: 13px;
 
+    letter-spacing: 0.1em;
 
+    text-transform: uppercase;
 
+    font-weight: 600;
 
+    color: var(--ink);
 
+    margin: 0;
 
+  }
 
+
 
+  p { margin: 0 0 16px; color: var(--ink); }
 
+  p.lead { font-size: 18px; color: var(--ink-dim); }
 
+  strong { color: var(--ink); }
 
+
 
+  ul.principles, ul.risks { list-style: none; margin: 0; padding: 0; display: grid; gap: 12px; }
 
+  ul.principles li, ul.risks li {
 
+    border: 1px solid var(--line);
 
+    background: var(--bg-alt);
 
+    border-radius: 10px;
 
+    padding: 14px 16px;
 
+    display: flex;
 
+    gap: 12px;
 
+    align-items: flex-start;
 
+  }
 
+  ul.principles li::before { content: "✓"; flex: 0 0 auto; width: 20px; color: var(--green); font-family: var(--mono); font-weight: 700; }
 
+  ul.risks li::before { content: "!"; flex: 0 0 auto; width: 20px; color: var(--red); font-family: var(--mono); font-weight: 700; text-align: center; }
 
+  ul.principles li .txt, ul.risks li .txt { flex: 1 1 auto; min-width: 0; }
 
+  ul.principles li .t, ul.risks li .t { font-weight: 700; color: var(--ink); display: block; margin-bottom: 3px; }
 
+  ul.principles li .d, ul.risks li .d { color: var(--ink-dim); font-size: 15.5px; }
 
+
 
+  .panel {
 
+    border: 1px solid var(--panel-line);
 
+    border-radius: 12px;
 
+    background: var(--bg-alt);
 
+    box-shadow: var(--shadow);
 
+    margin: 24px 0;
 
+    overflow: hidden;
 
+  }
 
+  .panel .bar {
 
+    display: flex;
 
+    align-items: center;
 
+    gap: 8px;
 
+    padding: 10px 16px;
 
+    border-bottom: 1px solid var(--panel-line);
 
+    font-family: var(--mono);
 
+    font-size: 11.5px;
 
+    letter-spacing: 0.06em;
 
+    text-transform: uppercase;
 
+    color: var(--ink-faint);
 
+  }
 
+  .panel .bar .dot { width: 7px; height: 7px; border-radius: 50%; background: var(--ink-faint); opacity: 0.5; }
 
+  .panel .body { padding: 18px; overflow-x: auto; }
 
+  .panel .body .mermaid { margin: 0; display: flex; justify-content: center; min-width: 560px; }
 
+
 
+  table {
 
+    width: 100%;
 
+    border-collapse: collapse;
 
+    font-size: 15px;
 
+  }
 
+  .table-scroll { overflow-x: auto; border: 1px solid var(--line); border-radius: 10px; }
 
+  table thead th {
 
+    font-family: var(--mono);
 
+    font-size: 11px;
 
+    letter-spacing: 0.07em;
 
+    text-transform: uppercase;
 
+    color: var(--ink-faint);
 
+    text-align: left;
 
+    padding: 12px 16px;
 
+    background: var(--bg-alt);
 
+    border-bottom: 1px solid var(--line);
 
+    white-space: nowrap;
 
+  }
 
+  table tbody td {
 
+    padding: 13px 16px;
 
+    border-bottom: 1px solid var(--line);
 
+    vertical-align: top;
 
+    color: var(--ink-dim);
 
+  }
 
+  table tbody tr:last-child td { border-bottom: none; }
 
+  table tbody td.field { font-family: var(--mono); font-size: 13.5px; color: var(--ink); white-space: nowrap; }
 
+  table code { font-family: var(--mono); font-size: 0.92em; background: var(--accent-soft); color: var(--accent); padding: 1px 6px; border-radius: 4px; }
 
+
 
+  .pill {
 
+    display: inline-block;
 
+    font-family: var(--mono);
 
+    font-size: 10.5px;
 
+    letter-spacing: 0.05em;
 
+    text-transform: uppercase;
 
+    padding: 3px 9px;
 
+    border-radius: 100px;
 
+    font-weight: 600;
 
+  }
 
+  .pill.open { background: var(--accent-soft); color: var(--accent); }
 
+  .pill.decided { background: var(--green-soft); color: var(--green); }
 
+
 
+  .boundary {
 
+    border: 1px dashed var(--panel-line);
 
+    border-radius: 12px;
 
+    padding: 20px 22px;
 
+    background: var(--bg-alt);
 
+  }
 
+  .boundary .t {
 
+    font-family: var(--mono);
 
+    font-size: 11px;
 
+    letter-spacing: 0.08em;
 
+    text-transform: uppercase;
 
+    color: var(--ink-faint);
 
+    margin-bottom: 8px;
 
+  }
 
+
 
+  footer {
 
+    margin-top: 80px;
 
+    padding-top: 24px;
 
+    border-top: 1px solid var(--line);
 
+    font-family: var(--mono);
 
+    font-size: 12px;
 
+    color: var(--ink-faint);
 
+    display: flex;
 
+    justify-content: space-between;
 
+    flex-wrap: wrap;
 
+    gap: 8px;
 
+  }
 
+
 
+  @media (max-width: 640px) {
 
+    nav.toc ol { columns: 1; }
 
+    .wrap { padding: 40px 18px 72px; }
 
+  }
 
+</style>
 
+</head>
 
+<body>
 
+<div class="wrap">
 
+
 
+  <div class="eyebrow">Internal discussion draft — no implementation started</div>
 
+  <h1>Field-to-Model: a correction&#8209;data pipeline for PalmOilAI</h1>
 
+  <p class="subtitle">Every grade a harvester corrects in the field is a labeled training example. Right now it never leaves the phone. This document proposes how it gets to us — quietly, reliably, and without asking the harvester to do anything.</p>
 
+
 
+  <div class="meta-row">
 
+    <span><b>Scope</b> — mobile app → dedicated ingestion server</span>
 
+    <span><b>Status</b> — brainstorm, pending decisions marked below</span>
 
+    <span><b>Owner</b> — palm_oil_mobile</span>
 
+  </div>
 
+
 
+  <nav class="toc" aria-label="Table of contents">
 
+    <p>Contents</p>
 
+    <ol>
 
+      <li><a href="#context"><span class="n">01</span>Why this exists</a></li>
 
+      <li><a href="#principles"><span class="n">02</span>Agreed principles</a></li>
 
+      <li><a href="#architecture"><span class="n">03</span>System architecture</a></li>
 
+      <li><a href="#flow"><span class="n">04</span>Sync flow, step by step</a></li>
 
+      <li><a href="#data"><span class="n">05</span>What actually gets sent</a></li>
 
+      <li><a href="#decisions"><span class="n">06</span>Open decisions</a></li>
 
+      <li><a href="#risks"><span class="n">07</span>Risks &amp; mitigations</a></li>
 
+      <li><a href="#boundary"><span class="n">08</span>Out of scope</a></li>
 
+    </ol>
 
+  </nav>
 
+
 
+  <section id="context">
 
+    <div class="sec-head"><span class="n">01</span><h2>Why this exists</h2></div>
 
+    <p class="lead">The app already lets a harvester correct the AI's grading — change a class, remove a false detection, or draw a box around a bunch the model missed. That correction is never destructive: the AI's original call is kept, and the human's edit sits alongside it.</p>
 
+    <p>That pairing — <em>what the model said</em> next to <em>what was actually true</em> — is exactly the labeled data needed to train the next generation of the model. Today it's trapped in a local SQLite database on each harvester's phone and goes nowhere. This document proposes a pipeline to collect it centrally, without adding a single step to the harvester's workflow.</p>
 
+  </section>
 
+
 
+  <section id="principles">
 
+    <div class="sec-head"><span class="n">02</span><h2>Agreed principles</h2></div>
 
+    <p>Three calls have already been made in discussion; everything downstream in this document follows from them.</p>
 
+    <ul class="principles">
 
+      <li>
 
+        <div class="txt">
 
+          <span class="t">A dedicated, standalone ingestion server</span>
 
+          <span class="d">Not bolted onto <code>server-desktop</code> or any operational service. If it's slow or down, nothing a harvester relies on in the field is affected — the data simply queues on-device until it's reachable again.</span>
 
+        </div>
 
+      </li>
 
+      <li>
 
+        <div class="txt">
 
+          <span class="t">Fully silent, background sync</span>
 
+          <span class="d">No "sync now" button, no progress bar, no prompt. The harvester's job is to scan and correct; getting that data to us is entirely the app's problem, running only when connectivity allows.</span>
 
+        </div>
 
+      </li>
 
+      <li>
 
+        <div class="txt">
 
+          <span class="t">The app tracks its own upload state</span>
 
+          <span class="d">Every record knows locally whether it's been sent. No handshake with the server is needed to know what's still pending — a local flag is the source of truth, which makes the whole thing resumable after any interruption.</span>
 
+        </div>
 
+      </li>
 
+    </ul>
 
+  </section>
 
+
 
+  <section id="architecture">
 
+    <div class="sec-head"><span class="n">03</span><h2>System architecture</h2></div>
 
+    <p>The phone is the only place data is created. Everything after that is designed so a slow or offline server never blocks a harvester from scanning.</p>
 
+    <div class="panel">
 
+      <div class="bar"><span class="dot"></span>figure 1 — data path from bunch to model</div>
 
+      <div class="body">
 
+<pre class="mermaid">
 
+flowchart LR
 
+  classDef device fill:#00000000,stroke-width:1px;
 
+  classDef server fill:#00000000,stroke-width:1px;
 
+  classDef future stroke-dasharray: 4 3;
 
+
 
+  H(["Harvester"]) --> P["Phone: scan + correct"]
 
+  P --> L[("Local SQLite\nscans + corrections\n+ upload state")]
 
+  L -- "background sync\nWi-Fi / connectivity trigger" --> S["Dedicated Ingestion Server"]
 
+  S --> R[("Raw Store\nimages + AI result + correction")]
 
+  R --> T["Retraining Pipeline\n(offline, periodic — future work)"]
 
+  T --> M["New Model Version"]
 
+  M -.-> P
 
+
 
+  class P,L device
 
+  class S,R server
 
+  class T,M future
 
+</pre>
 
+      </div>
 
+    </div>
 
+    <p>The dashed link back to the phone is deliberately out of scope for this document — how a new model version reaches devices is a separate rollout question for later.</p>
 
+  </section>
 
+
 
+  <section id="flow">
 
+    <div class="sec-head"><span class="n">04</span><h2>Sync flow, step by step</h2></div>
 
+    <p>The harvester only ever sees the left-most lane. Everything else happens whenever the phone next has connectivity, with no user-visible state.</p>
 
+    <div class="panel">
 
+      <div class="bar"><span class="dot"></span>figure 2 — from a correction to an uploaded record</div>
 
+      <div class="body">
 
+<pre class="mermaid">
 
+sequenceDiagram
 
+  participant H as Harvester
 
+  participant App as App
 
+  participant DB as Local DB
 
+  participant Sync as Background Sync
 
+  participant Srv as Ingestion Server
 
+
 
+  H->>App: Scan bunch, correct a grade
 
+  App->>DB: Save record (uploadedAt = none)
 
+  Note right of DB: Sits queued.<br/>Nothing for the harvester to do.
 
+  loop Whenever the OS wakes it
 
+    Sync->>DB: Which records are unsynced?
 
+    alt connectivity available
 
+      Sync->>Srv: Upload batch (image + AI result + correction)
 
+      Srv-->>Sync: Acknowledged
 
+      Sync->>DB: Mark uploadedAt = now
 
+    else no connectivity
 
+      Sync->>Sync: Do nothing, try again next wake
 
+    end
 
+  end
 
+</pre>
 
+      </div>
 
+    </div>
 
+    <p>One edge case worth naming: a harvester can correct a scan <em>after</em> it already uploaded — reviewing a batch a few days later, say. So "synced" needs to track the correction separately from the original scan, or a late correction silently never makes it off the device.</p>
 
+  </section>
 
+
 
+  <section id="data">
 
+    <div class="sec-head"><span class="n">05</span><h2>What actually gets sent</h2></div>
 
+    <p>Per scan, the payload pairs the model's original call with whatever the harvester changed it to — that pairing is the entire point of the exercise.</p>
 
+    <div class="table-scroll">
 
+      <table>
 
+        <thead>
 
+          <tr><th>Field</th><th>Where it comes from</th><th>Why it matters for training</th></tr>
 
+        </thead>
 
+        <tbody>
 
+          <tr><td class="field">image</td><td>Camera capture</td><td>The actual training input</td></tr>
 
+          <tr><td class="field">ai_detections[]</td><td>On-device inference</td><td>The model's original boxes, classes, confidence — what it got right or wrong</td></tr>
 
+          <tr><td class="field">human_correction[]</td><td>Harvester review</td><td>The corrected boxes/classes, or a removed false positive — the actual label</td></tr>
 
+          <tr><td class="field">model_version</td><td>App build</td><td>Which model produced the AI call, so improvement can be measured generation over generation</td></tr>
 
+          <tr><td class="field">captured_at</td><td>App</td><td>Chronological + seasonal context for the dataset</td></tr>
 
+          <tr><td class="field">device_id</td><td>App</td><td>Coarse debugging / dedup, not tied to a personal identity</td></tr>
 
+          <tr><td class="field">uploaded_at</td><td>Sync engine, local only</td><td>Never sent — this is the idempotency marker that makes sync resumable</td></tr>
 
+        </tbody>
 
+      </table>
 
+    </div>
 
+  </section>
 
+
 
+  <section id="decisions">
 
+    <div class="sec-head"><span class="n">06</span><h2>Open decisions</h2></div>
 
+    <p>Three things still need a call before this can move from brainstorm to plan.</p>
 
+    <div class="table-scroll">
 
+      <table>
 
+        <thead>
 
+          <tr><th>Question</th><th>Leaning</th><th>Trade-off</th><th></th></tr>
 
+        </thead>
 
+        <tbody>
 
+          <tr>
 
+            <td class="field">Wi-Fi only, or any connection?</td>
 
+            <td>Wi-Fi only to start</td>
 
+            <td>Slower data arrival, but no surprise mobile-data cost for a harvester who never asked for this</td>
 
+            <td><span class="pill open">Open</span></td>
 
+          </tr>
 
+          <tr>
 
+            <td class="field">Upload every scan, or only corrected ones?</td>
 
+            <td>Every reviewed scan</td>
 
+            <td>More storage, but "the AI was right, no correction needed" is itself a useful training signal, not noise</td>
 
+            <td><span class="pill open">Open</span></td>
 
+          </tr>
 
+          <tr>
 
+            <td class="field">Silent upload, or an opt-in toggle?</td>
 
+            <td>Depends on device ownership</td>
 
+            <td>Opt-in adds friction and drop-off; silent upload only sits right if these are company-managed devices</td>
 
+            <td><span class="pill open">Open</span></td>
 
+          </tr>
 
+          <tr>
 
+            <td class="field">Dedicated, standalone server</td>
 
+            <td>Decided</td>
 
+            <td>Keeps ingestion downtime from ever touching the app's core detection features</td>
 
+            <td><span class="pill decided">Decided</span></td>
 
+          </tr>
 
+          <tr>
 
+            <td class="field">Fully background, no harvester-facing sync UI</td>
 
+            <td>Decided</td>
 
+            <td>Zero added burden on the harvester's workflow</td>
 
+            <td><span class="pill decided">Decided</span></td>
 
+          </tr>
 
+        </tbody>
 
+      </table>
 
+    </div>
 
+  </section>
 
+
 
+  <section id="risks">
 
+    <div class="sec-head"><span class="n">07</span><h2>Risks &amp; mitigations</h2></div>
 
+    <ul class="risks">
 
+      <li>
 
+        <div class="txt">
 
+          <span class="t">Ingestion server is down or slow</span>
 
+          <span class="d">No effect on scanning or correcting — records simply stay queued locally and upload on the next successful sync. This is the entire reason it's a separate server.</span>
 
+        </div>
 
+      </li>
 
+      <li>
 
+        <div class="txt">
 
+          <span class="t">Bandwidth cost on a harvester's personal data plan</span>
 
+          <span class="d">Addressed by the Wi-Fi-only decision above, once confirmed.</span>
 
+        </div>
 
+      </li>
 
+      <li>
 
+        <div class="txt">
 
+          <span class="t">A correction made after the scan already synced gets missed</span>
 
+          <span class="d">Track correction-sync state separately from scan-sync state, so an edit made days later still gets picked up on the next background pass.</span>
 
+        </div>
 
+      </li>
 
+      <li>
 
+        <div class="txt">
 
+          <span class="t">Raw storage grows indefinitely</span>
 
+          <span class="d">Not a phone-side concern, but worth flagging for the server design — periodic archival into a proper training dataset, rather than an ever-growing inbox.</span>
 
+        </div>
 
+      </li>
 
+    </ul>
 
+  </section>
 
+
 
+  <section id="boundary">
 
+    <div class="sec-head"><span class="n">08</span><h2>Out of scope</h2></div>
 
+    <div class="boundary">
 
+      <div class="t">Where this document stops</div>
 
+      <p style="margin-bottom:0;">The app's responsibility ends at <strong>reliably getting corrected data to the server</strong>. Aggregating uploads into a training dataset, running retraining, evaluating a new model, and distributing it back to devices are separate initiatives — worth planning, but deliberately not decided here.</p>
 
+    </div>
 
+  </section>
 
+
 
+  <footer>
 
+    <span>Field-to-Model — correction data pipeline</span>
 
+    <span>Discussion draft, no code changes made</span>
 
+  </footer>
 
+
 
+</div>
 
+
 
+<script src="https://cdn.jsdelivr.net/npm/mermaid@10/dist/mermaid.min.js"></script>
 
+<script>
 
+  var prefersDark = window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches;
 
+  mermaid.initialize({ startOnLoad: true, theme: prefersDark ? 'dark' : 'default' });
 
+</script>
 
+</body>
 
+</html>

+ 105 - 0
DATA_COLLECTION_PIPELINE.md
View File 

@@ -0,0 +1,105 @@
 
+# Field-to-Model: Correction Data Pipeline
 
+
 
+> **Status:** Discussion draft — no implementation started, no code changes made.
 
+> **Scope:** Mobile app → dedicated ingestion server. Retraining itself is out of scope (see [Out of scope](#out-of-scope)).
 
+
 
+## Why this exists
 
+
 
+The app already lets a harvester correct the AI's grading — change a class, remove a false detection, or draw a box around a bunch the model missed. That correction is never destructive: the AI's original call is kept, and the human's edit sits alongside it (see the `history` / `corrections` tables in `lib/services/database_helper.dart`).
 
+
 
+That pairing — *what the model said* next to *what was actually true* — is exactly the labeled data needed to train the next generation of the model. Today it's trapped in a local SQLite database on each harvester's phone and goes nowhere. This document proposes a pipeline to collect it centrally, without adding a single step to the harvester's workflow.
 
+
 
+## Agreed principles
 
+
 
+Three calls have already been made in discussion; everything downstream follows from them.
 
+
 
+1. **A dedicated, standalone ingestion server.** Not bolted onto `server-desktop` or any operational service. If it's slow or down, nothing a harvester relies on in the field is affected — the data simply queues on-device until it's reachable again.
 
+2. **Fully silent, background sync.** No "sync now" button, no progress bar, no prompt. The harvester's job is to scan and correct; getting that data to us is entirely the app's problem, running only when connectivity allows.
 
+3. **The app tracks its own upload state.** Every record knows locally whether it's been sent. No handshake with the server is needed to know what's still pending — a local flag is the source of truth, which makes the whole thing resumable after any interruption.
 
+
 
+## System architecture
 
+
 
+The phone is the only place data is created. Everything after that is designed so a slow or offline server never blocks a harvester from scanning.
 
+
 
+```mermaid
 
+flowchart LR
 
+  H(["Harvester"]) --> P["Phone: scan + correct"]
 
+  P --> L[("Local SQLite\nscans + corrections\n+ upload state")]
 
+  L -- "background sync\nWi-Fi / connectivity trigger" --> S["Dedicated Ingestion Server"]
 
+  S --> R[("Raw Store\nimages + AI result + correction")]
 
+  R --> T["Retraining Pipeline\n(offline, periodic — future work)"]
 
+  T --> M["New Model Version"]
 
+  M -.-> P
 
+```
 
+
 
+The dashed link back to the phone is deliberately out of scope for this document — how a new model version reaches devices is a separate rollout question for later.
 
+
 
+## Sync flow, step by step
 
+
 
+The harvester only ever sees the left-most lane. Everything else happens whenever the phone next has connectivity, with no user-visible state.
 
+
 
+```mermaid
 
+sequenceDiagram
 
+  participant H as Harvester
 
+  participant App as App
 
+  participant DB as Local DB
 
+  participant Sync as Background Sync
 
+  participant Srv as Ingestion Server
 
+
 
+  H->>App: Scan bunch, correct a grade
 
+  App->>DB: Save record (uploadedAt = none)
 
+  Note right of DB: Sits queued.<br/>Nothing for the harvester to do.
 
+  loop Whenever the OS wakes it
 
+    Sync->>DB: Which records are unsynced?
 
+    alt connectivity available
 
+      Sync->>Srv: Upload batch (image + AI result + correction)
 
+      Srv-->>Sync: Acknowledged
 
+      Sync->>DB: Mark uploadedAt = now
 
+    else no connectivity
 
+      Sync->>Sync: Do nothing, try again next wake
 
+    end
 
+  end
 
+```
 
+
 
+One edge case worth naming: a harvester can correct a scan *after* it already uploaded — reviewing a batch a few days later, say. So "synced" needs to track the correction separately from the original scan, or a late correction silently never makes it off the device.
 
+
 
+## What actually gets sent
 
+
 
+Per scan, the payload pairs the model's original call with whatever the harvester changed it to — that pairing is the entire point of the exercise.
 
+
 
+| Field | Where it comes from | Why it matters for training |
 
+|---|---|---|
 
+| `image` | Camera capture | The actual training input |
 
+| `ai_detections[]` | On-device inference | The model's original boxes, classes, confidence — what it got right or wrong |
 
+| `human_correction[]` | Harvester review | The corrected boxes/classes, or a removed false positive — the actual label |
 
+| `model_version` | App build | Which model produced the AI call, so improvement can be measured generation over generation |
 
+| `captured_at` | App | Chronological + seasonal context for the dataset |
 
+| `device_id` | App | Coarse debugging / dedup, not tied to a personal identity |
 
+| `uploaded_at` | Sync engine, local only | Never sent — this is the idempotency marker that makes sync resumable |
 
+
 
+## Open decisions
 
+
 
+Three things still need a call before this can move from brainstorm to plan.
 
+
 
+| Question | Leaning | Trade-off | |
 
+|---|---|---|---|
 
+| Wi-Fi only, or any connection? | Wi-Fi only to start | Slower data arrival, but no surprise mobile-data cost for a harvester who never asked for this | **Open** |
 
+| Upload every scan, or only corrected ones? | Every reviewed scan | More storage, but "the AI was right, no correction needed" is itself a useful training signal, not noise | **Open** |
 
+| Silent upload, or an opt-in toggle? | Depends on device ownership | Opt-in adds friction and drop-off; silent upload only sits right if these are company-managed devices | **Open** |
 
+| Dedicated, standalone server | — | Keeps ingestion downtime from ever touching the app's core detection features | **Decided** |
 
+| Fully background, no harvester-facing sync UI | — | Zero added burden on the harvester's workflow | **Decided** |
 
+
 
+## Risks & mitigations
 
+
 
+- **Ingestion server is down or slow.** No effect on scanning or correcting — records simply stay queued locally and upload on the next successful sync. This is the entire reason it's a separate server.
 
+- **Bandwidth cost on a harvester's personal data plan.** Addressed by the Wi-Fi-only decision above, once confirmed.
 
+- **A correction made after the scan already synced gets missed.** Track correction-sync state separately from scan-sync state, so an edit made days later still gets picked up on the next background pass.
 
+- **Raw storage grows indefinitely.** Not a phone-side concern, but worth flagging for the server design — periodic archival into a proper training dataset, rather than an ever-growing inbox.
 
+
 
+## Out of scope
 
+
 
+The app's responsibility ends at **reliably getting corrected data to the server**. Aggregating uploads into a training dataset, running retraining, evaluating a new model, and distributing it back to devices are separate initiatives — worth planning, but deliberately not decided here.
 
+
 
+---
 
+
 
+*Companion presentation version (with rendered diagrams) prepared as a shareable artifact for stakeholder review.*