/` HTML skeleton (empty cells) and
+ * the raw structure token ids it was built from (start/end tokens stripped).
+ * @category Types
+ */
+export type TableStructure = {
+ readonly html: string;
+ readonly tokens: number[];
+};
+
+/**
+ * Configuration for the fused document models — page orientation, geometric
+ * dewarp, and table-structure recognition, all exposed by one model file.
+ * `structureVocab` maps a table token id (array index) to its HTML fragment;
+ * `eosTokenId` ends table decoding and `maxSteps` caps it.
+ * @category Types
+ */
+export type DocumentModelsConfig = {
+ readonly modelPath: string;
+ readonly structureVocab: readonly string[];
+ readonly eosTokenId: number;
+ readonly maxSteps: number;
+};
+
+// A dewarp grid estimated on a page without clear boundaries (e.g. text floating
+// on white) can map most of the output off the source, collapsing the page to
+// near-blank and OCR to zero detections. The dewarp guard compares content before
+// and after: if the warp keeps less than this fraction of the source's activity,
+// it is declined and the original page is kept.
+const DEWARP_MIN_ACTIVITY_RATIO = 0.5;
+const DEWARP_ACTIVITY_STRIDE = 31;
+
+// The variance of one channel, sampled every DEWARP_ACTIVITY_STRIDE pixels — a
+// cheap, polarity-independent proxy for how much content (ink/edges) an image
+// carries; a blank page is ~0. Used by the dewarp degeneracy guard.
+function dewarpActivity(data: Uint8Array, channels: number): number {
+ 'worklet';
+ let n = 0;
+ let sum = 0;
+ let sumSq = 0;
+ const step = channels * DEWARP_ACTIVITY_STRIDE;
+ for (let i = 0; i < data.length; i += step) {
+ const v = data[i]!;
+ sum += v;
+ sumSq += v * v;
+ n++;
+ }
+ if (n === 0) {
+ return 0;
+ }
+ const mean = sum / n;
+ return sumSq / n - mean * mean;
+}
+
+// Index of the maximum value in `arr[offset, offset+len)` (single pass, no allocation).
+function argmaxRange(arr: ArrayLike, offset: number, len: number): number {
+ 'worklet';
+ let index = 0;
+ let best = arr[offset]!;
+ for (let i = 1; i < len; i++) {
+ const value = arr[offset + i]!;
+ if (value > best) {
+ best = value;
+ index = i;
+ }
+ }
+ return index;
+}
+
+// Assembles the table-structure content tokens into an HTML skeleton, dropping the
+// reserved start/end range.
+function tokensToHtml(
+ tokens: number[],
+ structureVocab: readonly string[],
+ eosTokenId: number
+): string {
+ 'worklet';
+ let html = '';
+ for (const t of tokens) {
+ if (t > 0 && t < eosTokenId && t < structureVocab.length) {
+ html += structureVocab[t]!;
+ }
+ }
+ return html;
+}
+
+/**
+ * Creates the document-models runner: page orientation, geometric dewarp (applied
+ * via the predicted sampling grid), and table-structure recognition (autoregressive
+ * decode). One model file, loaded once. Internal to the document pipeline.
+ * @category Typescript API
+ * @param config Model path, table-structure vocabulary, and decode limits.
+ * @param runtime Optional worklet runtime thread.
+ * @returns A promise resolving to the three capabilities plus disposal controls.
+ */
+export async function createDocumentModels(
+ config: DocumentModelsConfig,
+ runtime?: WorkletRuntime
+): Promise<{
+ dispose: () => void;
+ detectOrientation: (page: Tensor, format: ImageFormat) => Promise;
+ detectOrientationWorklet: (page: Tensor, format: ImageFormat) => Orientation;
+ dewarp: (page: Tensor, format: ImageFormat) => Promise;
+ dewarpWorklet: (page: Tensor, format: ImageFormat) => Tensor;
+ recognizeTable: (input: ImageBuffer) => Promise;
+ recognizeTableWorklet: (input: ImageBuffer) => TableStructure;
+}> {
+ const { modelPath, structureVocab, eosTokenId, maxSteps } = config;
+ const model = await wrapAsync(loadModel, runtime)(modelPath);
+
+ // Everything built is pushed into `created` as it is created — one by one, so
+ // a mid-sequence failure can't strand its predecessors — and the catch
+ // disposes it all: a bad config must not leak native memory (mirrors createOcr).
+ const created: { dispose: () => void }[] = [];
+ try {
+ // orientation: image -> class logits
+ const oriMeta = validateModelSchema(
+ model,
+ 'orientation',
+ [SymbolicTensor('float32', [1, 3, 'H', 'W'])],
+ [SymbolicTensor('float32', [1, 'K'])]
+ );
+ // dewarp: image -> sampling grid
+ const dewMeta = validateModelSchema(
+ model,
+ 'dewarp',
+ [SymbolicTensor('float32', [1, 3, 'H', 'W'])],
+ [SymbolicTensor('float32', [1, 2, 'gH', 'gW'])]
+ );
+ // table: image encoder + autoregressive decode step
+ const encMeta = validateModelSchema(
+ model,
+ 'table_encode',
+ [SymbolicTensor('float32', [1, 3, 'H', 'W'])],
+ [SymbolicTensor('float32', [1, 'C', 'F'])]
+ );
+ const decMeta = validateModelSchema(
+ model,
+ 'table_decode_step',
+ [
+ SymbolicTensor('float32', [1, 'C', 'F']),
+ SymbolicTensor('float32', [1, 'H']),
+ SymbolicTensor('float32', [1, 'V']),
+ ],
+ [SymbolicTensor('float32', [1, 'V']), SymbolicTensor('float32', [1, 'H'])]
+ );
+
+ const oriShape = oriMeta.inputTensorMeta[0]!.shape;
+ const oriOutLen = oriMeta.outputTensorMeta[0]!.shape[1]!;
+ const dewShape = dewMeta.inputTensorMeta[0]!.shape;
+ const gridShape = dewMeta.outputTensorMeta[0]!.shape; // [1,2,gH,gW]
+ const tabShape = encMeta.inputTensorMeta[0]!.shape;
+ const featShape = encMeta.outputTensorMeta[0]!.shape;
+ const hidShape = decMeta.outputTensorMeta[1]!.shape;
+ const probShape = decMeta.outputTensorMeta[0]!.shape;
+ const hidLen = hidShape[1]!;
+ const vocabLen = probShape[1]!;
+
+ if (vocabLen !== structureVocab.length) {
+ throw new Error(
+ `DocumentModels: structure vocab length (${structureVocab.length}) must match the model's token dim (${vocabLen}).`
+ );
+ }
+
+ const orientationPreprocessor = createImagePreprocessor(
+ {
+ resizeMode: 'stretch',
+ interpolation: 'linear',
+ alpha: IMAGENET_NORM.alpha,
+ beta: IMAGENET_NORM.beta,
+ },
+ oriShape
+ );
+ created.push(orientationPreprocessor);
+ const dewarpPreprocessor = createImagePreprocessor(
+ { resizeMode: 'stretch', interpolation: 'linear', alpha: 1 / 255, beta: 0 },
+ dewShape
+ );
+ created.push(dewarpPreprocessor);
+ const tablePreprocessor = createImagePreprocessor(
+ {
+ resizeMode: 'stretch',
+ interpolation: 'linear',
+ alpha: IMAGENET_NORM.alpha,
+ beta: IMAGENET_NORM.beta,
+ },
+ tabShape
+ );
+ created.push(tablePreprocessor);
+
+ const tOri = tensor('float32', oriMeta.outputTensorMeta[0]!.shape);
+ created.push(tOri);
+ const tGrid = tensor('float32', gridShape);
+ created.push(tGrid);
+ const tFeatures = tensor('float32', featShape);
+ created.push(tFeatures);
+ const tHidden = tensor('float32', hidShape);
+ created.push(tHidden);
+ const tOnehot = tensor('float32', probShape);
+ created.push(tOnehot);
+ const tProbs = tensor('float32', probShape);
+ created.push(tProbs);
+ const tNewHidden = tensor('float32', hidShape);
+ created.push(tNewHidden);
+
+ const oriBuf = new Float32Array(oriOutLen);
+ const zeroHidden = new Float32Array(hidLen);
+ const zeroVocab = new Float32Array(vocabLen);
+ const onehotBuf = new Float32Array(vocabLen);
+ const probsBuf = new Float32Array(vocabLen);
+
+ const dispose = () => {
+ created.forEach((c) => c.dispose());
+ model.dispose();
+ };
+
+ const detectOrientationWorklet = (page: Tensor, format: ImageFormat): Orientation => {
+ 'worklet';
+ const tInput = orientationPreprocessor.processTensor(page, format);
+ model.execute('orientation', [tInput], [tOri]);
+ tOri.getData(oriBuf);
+ const cls = argmaxRange(oriBuf, 0, oriOutLen);
+ const best = oriBuf[cls]!;
+ let sumExp = 0;
+ for (let i = 0; i < oriOutLen; i++) {
+ sumExp += Math.exp(oriBuf[i]! - best);
+ }
+ const rotationCW = ((cls % 4) * 90) as 0 | 90 | 180 | 270;
+ const confidence = 1 / sumExp;
+ return { rotationCW, confidence };
+ };
+
+ // Dewarps the full-res page tensor in place: estimate the sampling field, apply
+ // it natively (cv::remap). Returns the dewarped tensor, or the input `page`
+ // unchanged when the warp is declined (caller owns whichever is returned).
+ const dewarpWorklet = (page: Tensor, format: ImageFormat): Tensor => {
+ 'worklet';
+ const tInput = dewarpPreprocessor.processTensor(page, format);
+ model.execute('dewarp', [tInput], [tGrid]);
+ const h = page.shape[0]!;
+ const w = page.shape[1]!;
+ const ch = FORMAT_CHANNELS[format];
+ const tDst = tensor('uint8', [h, w, ch]);
+ try {
+ warpByGrid(page, tGrid, tDst);
+ const out = new Uint8Array(w * h * ch);
+ const src = new Uint8Array(w * h * ch);
+ tDst.getData(out);
+ page.getData(src);
+ // Degenerate-warp guard: a grid lacking page boundaries can push content
+ // off-canvas, leaving a near-blank page. If the dewarp collapsed the image's
+ // activity, decline it and keep the original (better an un-dewarped read than
+ // zero detections).
+ if (dewarpActivity(out, ch) < DEWARP_MIN_ACTIVITY_RATIO * dewarpActivity(src, ch)) {
+ tDst.dispose();
+ return page;
+ }
+ return tDst;
+ } catch (e) {
+ // On failure the caller can't see tDst to free it (success path returns it),
+ // so release it here before propagating.
+ tDst.dispose();
+ throw e;
+ }
+ };
+
+ const recognizeTableWorklet = (input: ImageBuffer): TableStructure => {
+ 'worklet';
+ const tInput = tablePreprocessor.process(input);
+ model.execute('table_encode', [tInput], [tFeatures]);
+ tHidden.setData(zeroHidden);
+ tOnehot.setData(zeroVocab);
+ const tokens: number[] = [];
+ for (let step = 0; step < maxSteps; step++) {
+ model.execute('table_decode_step', [tFeatures, tHidden, tOnehot], [tProbs, tNewHidden]);
+ tProbs.getData(probsBuf);
+ const tok = argmaxRange(probsBuf, 0, vocabLen);
+ tokens.push(tok);
+ if (tok === eosTokenId) {
+ break;
+ }
+ tNewHidden.copyTo(tHidden);
+ onehotBuf.fill(0);
+ onehotBuf[tok] = 1;
+ tOnehot.setData(onehotBuf);
+ }
+ return { html: tokensToHtml(tokens, structureVocab, eosTokenId), tokens };
+ };
+
+ return {
+ dispose,
+ detectOrientation: wrapAsync(detectOrientationWorklet, runtime),
+ detectOrientationWorklet,
+ dewarp: wrapAsync(dewarpWorklet, runtime),
+ dewarpWorklet,
+ recognizeTable: wrapAsync(recognizeTableWorklet, runtime),
+ recognizeTableWorklet,
+ };
+ } catch (e) {
+ created.forEach((c) => c.dispose());
+ model.dispose();
+ throw e;
+ }
+}
+
+/**
+ * Crops an axis-aligned region out of an image as a plain pixel slice (same format
+ * and layout). Used to feed a layout region to another model.
+ * @category Typescript API
+ * @param input The source image.
+ * @param bbox The crop region, in `xyxy` pixels.
+ * @returns The cropped image.
+ */
+export function cropImageBuffer(input: ImageBuffer, bbox: BoundingBox<'xyxy'>): ImageBuffer {
+ 'worklet';
+ const { data, width, height, format } = input;
+ const channels = FORMAT_CHANNELS[format];
+ const x0 = Math.max(0, Math.min(Math.round(bbox.xmin), width));
+ const y0 = Math.max(0, Math.min(Math.round(bbox.ymin), height));
+ const x1 = Math.max(0, Math.min(Math.round(bbox.xmax), width));
+ const y1 = Math.max(0, Math.min(Math.round(bbox.ymax), height));
+ const cropWidth = Math.max(1, x1 - x0);
+ const cropHeight = Math.max(1, y1 - y0);
+ const out = new Uint8Array(cropWidth * cropHeight * channels);
+ for (let y = 0; y < cropHeight; y++) {
+ const rowStart = ((y0 + y) * width + x0) * channels;
+ out.set(data.subarray(rowStart, rowStart + cropWidth * channels), y * cropWidth * channels);
+ }
+ return { data: out, width: cropWidth, height: cropHeight, format, layout: input.layout };
+}
+
+// 1-D clustering of cell-center coordinates into `k` table rows (or columns).
+// The sorted values are split at their k-1 widest gaps — each resulting run of
+// values is one row/column, represented by its mean coordinate. Splitting at the
+// widest gaps (instead of at fixed intervals) matches how table cells actually
+// distribute: values within a row are tightly packed while rows are separated by
+// clear gaps, so uneven row heights / column widths still cluster correctly.
+// Fewer than `k` values means every value is its own cluster.
+function clusterCentersByGaps(values: readonly number[], k: number): number[] {
+ 'worklet';
+ const sorted = [...values].sort((a, b) => a - b);
+ if (sorted.length <= k) {
+ return sorted;
+ }
+ // Rank the interior gaps (gap i sits between sorted[i-1] and sorted[i]) and take
+ // the k-1 widest as cut points, restored to ascending order.
+ const gaps = sorted.slice(1).map((value, i) => ({ at: i + 1, size: value - sorted[i]! }));
+ gaps.sort((a, b) => b.size - a.size);
+ const cuts = gaps
+ .slice(0, k - 1)
+ .map((gap) => gap.at)
+ .sort((a, b) => a - b);
+ // Average each [prev, cut) span into its center.
+ const centers: number[] = [];
+ let prev = 0;
+ for (const cut of [...cuts, sorted.length]) {
+ const group = sorted.slice(prev, cut);
+ centers.push(group.reduce((sum, value) => sum + value, 0) / group.length);
+ prev = cut;
+ }
+ return centers;
+}
+
+/**
+ * Fills a table-structure HTML skeleton with a region's OCR lines. The grid size
+ * comes from the skeleton (row count, and the widest row's cell count); each
+ * line's box center is assigned to its nearest row and column cluster, so shared
+ * column centers keep columns aligned. Falls back to a document-order fill when the
+ * skeleton has no grid.
+ *
+ * Alignment is geometric only — dense rows can misplace a value, since the
+ * skeleton carries no per-cell coordinates.
+ * @category Typescript API
+ * @param html The structure HTML skeleton (empty cells).
+ * @param lines The region's OCR lines, with page-space quads.
+ * @returns A `` with each cell filled by its nearest-assigned text.
+ */
+export function fillTableCells(html: string, lines: readonly OcrDetection[]): string {
+ 'worklet';
+ const rowCount = (html.match(//g) ?? []).length;
+ let colCount = 0;
+ const rowRegex = / ([\s\S]*?)<\/tr>/g;
+ let row: RegExpExecArray | null;
+ while ((row = rowRegex.exec(html)) !== null) {
+ colCount = Math.max(colCount, (row[1]!.match(/| ]*)><\/td>/g, (_match, attrs) => {
+ const text = i < lines.length ? lines[i]!.text : '';
+ i++;
+ return `${text} | `;
+ });
+ }
+
+ const centersX: number[] = [];
+ const centersY: number[] = [];
+ for (const line of lines) {
+ const box = boundsOfPoints(line.quad, 'xyxy');
+ centersX.push((box.xmin + box.xmax) / 2);
+ centersY.push((box.ymin + box.ymax) / 2);
+ }
+ const rowCenters = clusterCentersByGaps(centersY, rowCount);
+ const colCenters = clusterCentersByGaps(centersX, colCount);
+ const grid: string[][] = Array.from({ length: rowCenters.length }, () =>
+ new Array(colCenters.length).fill('')
+ );
+ // Assign each line to the row/column whose cluster center is nearest.
+ for (let i = 0; i < lines.length; i++) {
+ const r = rowCenters.reduce(
+ (best, center, j) =>
+ Math.abs(centersY[i]! - center) < Math.abs(centersY[i]! - rowCenters[best]!) ? j : best,
+ 0
+ );
+ const c = colCenters.reduce(
+ (best, center, j) =>
+ Math.abs(centersX[i]! - center) < Math.abs(centersX[i]! - colCenters[best]!) ? j : best,
+ 0
+ );
+ grid[r]![c] = `${grid[r]![c]!} ${lines[i]!.text}`.trim();
+ }
+ return `${grid
+ .map((cells) => `${cells.map((text) => `| ${text} | `).join('')} `)
+ .join('')} `;
+}
diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/ocr/documentOcr.ts b/packages/react-native-executorch/src/extensions/cv/tasks/ocr/documentOcr.ts
new file mode 100644
index 0000000000..6736432b48
--- /dev/null
+++ b/packages/react-native-executorch/src/extensions/cv/tasks/ocr/documentOcr.ts
@@ -0,0 +1,286 @@
+import type { WorkletRuntime } from 'react-native-worklets';
+
+import { tensor } from '../../../../core/tensor';
+import { wrapAsync } from '../../../../core/runtime';
+import type { ImageBuffer } from '../../image';
+import type { Point } from '../../ops/points';
+import type { BoundingBox } from '../../ops/boxes';
+import { boundsOfPoints } from '../../ops/quad';
+import { rotate, FORMAT_CHANNELS } from '../../ops/image';
+import { createOcr, type OcrModel, type OcrDetection } from './ocr';
+import {
+ createObjectDetector,
+ type ObjectDetectorModel,
+ type ObjectDetection,
+} from '../objectDetection';
+import {
+ createDocumentModels,
+ cropImageBuffer,
+ fillTableCells,
+ type DocumentModelsConfig,
+} from './documentModels';
+import { orderByReadingOrder } from './ocrUtils';
+
+/**
+ * One assembled document block: a layout region (or an ungrouped catch-all) with
+ * its OCR lines grouped and concatenated, in reading order.
+ * @category Types
+ */
+export type DocumentBlock = {
+ /** Region class from layout (e.g. `'text'`, `'table'`), or `'ungrouped'`. */
+ readonly regionType: L | 'ungrouped';
+ /** Block box in processing-frame pixels. */
+ readonly bbox: BoundingBox<'xyxy'>;
+ /** Layout confidence for the region (1 for `'ungrouped'`). */
+ readonly score: number;
+ /** The block's text, lines joined top-to-bottom by newlines. */
+ readonly text: string;
+ /** The OCR lines inside this block, top-to-bottom. */
+ readonly lines: readonly OcrDetection[];
+ /** Whether this block is a table region. */
+ readonly isTable: boolean;
+ /** For table blocks: the recognized HTML structure with OCR text filled in. */
+ readonly tableHtml?: string;
+};
+
+/**
+ * The result of a document OCR run.
+ * @category Types
+ */
+export type DocumentResult = {
+ readonly blocks: DocumentBlock[];
+ readonly regions: ObjectDetection<'xyxy', L>[];
+ readonly detections: OcrDetection[];
+ /**
+ * The frame all `bbox`/`quad` coordinates are relative to. Equals the input
+ * image unless orientation correction or dewarp was applied, in which case it
+ * is the corrected image — overlay boxes on THIS, not the original input.
+ */
+ readonly image: ImageBuffer;
+};
+
+/**
+ * Configuration for the document OCR orchestrator. Provides an OCR model, an
+ * optional layout model (regions/blocks), and optional document models
+ * (orientation/dewarp pre-processing + table-structure recognition). The
+ * `orientation`/`dewarp` flags are *defaults* for the per-run options of the
+ * same name — supply them here to bias every run, or leave them off and pass
+ * them to `runDocumentOcr` per call (the document models are loaded either way).
+ * @category Types
+ */
+export type DocumentOcrModel = {
+ readonly ocr: OcrModel;
+ readonly layout?: ObjectDetectorModel<'xyxy', L>;
+ readonly documentModels?: DocumentModelsConfig;
+ /** Default for the per-run `orientation` option (needs `documentModels`). */
+ readonly orientation?: boolean;
+ /** Default for the per-run `dewarp` option (needs `documentModels`). */
+ readonly dewarp?: boolean;
+ /**
+ * Minimum orientation-classifier confidence (softmax of the argmax class) to act
+ * on a non-zero rotation — below it the page is treated as already upright, so
+ * out-of-distribution inputs (photos/non-documents) don't spuriously flip. Genuine
+ * documents score >0.95; defaults to 0.85.
+ */
+ readonly orientationMinConfidence?: number;
+};
+
+/**
+ * Per-run document options (passed to `runDocumentOcr`, not baked into the
+ * model — toggling them needs no reload). Each pre-processing pass still
+ * requires the document models to have been loaded (`config.documentModels`).
+ * @category Types
+ */
+export type RunDocumentOcrOptions = {
+ /**
+ * Detect + correct page orientation before OCR. No-op without loaded document
+ * models. Defaults to the model's `config.orientation`.
+ */
+ readonly orientation?: boolean;
+ /**
+ * Geometrically dewarp the page before OCR. No-op without loaded document
+ * models. Defaults to the model's `config.dewarp`.
+ */
+ readonly dewarp?: boolean;
+};
+
+// Layout classes that carry no text — skip OCR on them.
+const VISUAL_LABELS = ['image', 'chart', 'seal'];
+
+function makeBlock(
+ regionType: L | 'ungrouped',
+ bbox: BoundingBox<'xyxy'>,
+ score: number,
+ lines: OcrDetection[],
+ isTable: boolean
+): DocumentBlock {
+ 'worklet';
+ const sorted = orderByReadingOrder(lines);
+ return {
+ regionType,
+ bbox,
+ score,
+ isTable,
+ lines: sorted,
+ text: sorted.map((l) => l.text).join('\n'),
+ };
+}
+
+/**
+ * Creates the document OCR orchestrator. Pipeline: correct orientation → dewarp
+ * (document models) → layout → per-region OCR (each text region is cropped and OCR'd on
+ * its own, upscaled into the detector — far better recall on dense pages than one
+ * whole-page pass; lines are offset back to page coords) → tables recognize their
+ * structure and fill cells with that region's OCR. Visual regions are skipped.
+ * Without layout it OCRs the whole page into one block. Layout/document models optional.
+ * @category Typescript API
+ * @param config OCR model + optional layout + optional document models + flags.
+ * @param runtime Optional worklet runtime thread.
+ * @returns A promise resolving to run + disposal controls.
+ */
+export async function createDocumentOcr(
+ config: DocumentOcrModel,
+ runtime?: WorkletRuntime
+): Promise<{
+ dispose: () => void;
+ runDocumentOcr: (
+ input: ImageBuffer,
+ options?: RunDocumentOcrOptions
+ ) => Promise>;
+ runDocumentOcrWorklet: (input: ImageBuffer, options?: RunDocumentOcrOptions) => DocumentResult;
+}> {
+ const ocr = await createOcr(config.ocr, runtime);
+ let layout: Awaited>> | null = null;
+ let documentModels: Awaited> | null = null;
+ try {
+ layout = config.layout ? await createObjectDetector<'xyxy', L>(config.layout, runtime) : null;
+ documentModels = config.documentModels
+ ? await createDocumentModels(config.documentModels, runtime)
+ : null;
+ } catch (e) {
+ // A later model failing to build must not leak the ones already built.
+ layout?.dispose();
+ ocr.dispose();
+ throw e;
+ }
+ const defaultOrientation = !!config.orientation;
+ const defaultDewarp = !!config.dewarp;
+ const minConfidence = config.orientationMinConfidence ?? 0.85;
+
+ const dispose = () => {
+ ocr.dispose();
+ layout?.dispose();
+ documentModels?.dispose();
+ };
+
+ const runDocumentOcrWorklet = (
+ input: ImageBuffer,
+ options?: RunDocumentOcrOptions
+ ): DocumentResult => {
+ 'worklet';
+ const useOrientation = !!documentModels && (options?.orientation ?? defaultOrientation);
+ const useDewarp = !!documentModels && (options?.dewarp ?? defaultDewarp);
+ let img = input;
+ if ((useOrientation || useDewarp) && documentModels) {
+ const ch = FORMAT_CHANNELS[input.format];
+ let page = tensor('uint8', [input.height, input.width, ch]);
+ page.setData(input.data);
+ let pw = input.width;
+ let ph = input.height;
+ try {
+ if (useOrientation) {
+ const orientation = documentModels.detectOrientationWorklet(page, input.format);
+ const deg = ((360 - orientation.rotationCW) % 360) as 0 | 90 | 180 | 270;
+ if (deg !== 0 && orientation.confidence >= minConfidence) {
+ const swap = deg === 90 || deg === 270;
+ const rotated = tensor('uint8', [swap ? pw : ph, swap ? ph : pw, ch]);
+ try {
+ rotate(page, rotated, deg);
+ } catch (e) {
+ rotated.dispose(); // rotate threw before we adopted `rotated` as `page`
+ throw e;
+ }
+ page.dispose();
+ page = rotated;
+ if (swap) {
+ [pw, ph] = [ph, pw];
+ }
+ }
+ }
+ if (useDewarp) {
+ // dewarp returns the input tensor unchanged when it declines the warp.
+ const dewarped = documentModels.dewarpWorklet(page, input.format);
+ if (dewarped !== page) {
+ page.dispose();
+ page = dewarped;
+ }
+ }
+ const out = new Uint8Array(pw * ph * ch);
+ page.getData(out);
+ img = { data: out, width: pw, height: ph, format: input.format, layout: input.layout };
+ } finally {
+ page.dispose();
+ }
+ }
+
+ try {
+ if (!layout) {
+ const detections = ocr.runOcrWorklet(img, { release: false }).detections;
+ const blocks = detections.length
+ ? [
+ makeBlock(
+ 'ungrouped',
+ boundsOfPoints(
+ detections.flatMap((d) => d.quad as Point[]),
+ 'xyxy'
+ ),
+ 1,
+ detections,
+ false
+ ),
+ ]
+ : [];
+ return { blocks, regions: [], detections, image: img };
+ }
+
+ const regions = layout.detectObjectsWorklet(img);
+ const blocks: DocumentBlock[] = [];
+ const detections: OcrDetection[] = [];
+ for (const region of regions) {
+ if (VISUAL_LABELS.includes(String(region.label))) {
+ continue;
+ }
+ const { xmin, ymin } = region.box;
+ const crop = cropImageBuffer(img, region.box);
+ const lines = ocr.runOcrWorklet(crop, { release: false }).detections.map((d) => ({
+ ...d,
+ quad: d.quad.map((p) => ({ x: p.x + xmin, y: p.y + ymin })),
+ }));
+ if (lines.length === 0 && region.label !== 'table') {
+ continue;
+ }
+ detections.push(...lines);
+ let block = makeBlock(
+ region.label,
+ region.box,
+ region.confidence,
+ lines,
+ region.label === 'table'
+ );
+ if (region.label === 'table' && documentModels) {
+ const structure = documentModels.recognizeTableWorklet(crop);
+ block = { ...block, tableHtml: fillTableCells(structure.html, block.lines) };
+ }
+ blocks.push(block);
+ }
+ blocks.sort((a, b) => a.bbox.ymin - b.bbox.ymin || a.bbox.xmin - b.bbox.xmin);
+ return { blocks, regions, detections, image: img };
+ } finally {
+ // Per-region runs pass release: false; the bucket arenas are freed once per page.
+ ocr.releaseMethodsWorklet();
+ }
+ };
+
+ const runDocumentOcr = wrapAsync(runDocumentOcrWorklet, runtime);
+ return { runDocumentOcr, runDocumentOcrWorklet, dispose };
+}
diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/ocr/ocr.ts b/packages/react-native-executorch/src/extensions/cv/tasks/ocr/ocr.ts
new file mode 100644
index 0000000000..d1263873b1
--- /dev/null
+++ b/packages/react-native-executorch/src/extensions/cv/tasks/ocr/ocr.ts
@@ -0,0 +1,405 @@
+import type { WorkletRuntime } from 'react-native-worklets';
+
+import { tensor, type Tensor } from '../../../../core/tensor';
+import { loadModel } from '../../../../core/model';
+import { wrapAsync } from '../../../../core/runtime';
+
+import type { ImageBuffer } from '../../image';
+import type { Point } from '../../ops/points';
+import { FORMAT_CHANNELS, FORMAT_CONVERSION, cvtColor } from '../../ops/image';
+import { orderQuad, quadSize, boundingQuadOf } from '../../ops/quad';
+import type { TextBoxExtractor } from './detectors';
+import { orderByReadingOrder, groupVerticalColumns, type Buckets } from './ocrUtils';
+import {
+ detectQuads,
+ recognizeQuad,
+ recognizeGlyphStrip,
+ readStackedColumn,
+ resolveDetectorContract,
+ resolveRecognizerContract,
+ type DetSet,
+ type RecSet,
+ type DetectContext,
+ type RecContext,
+ type VerticalContext,
+} from './pipeline';
+
+export type { Buckets } from './ocrUtils';
+export type { Quad } from '../../ops/quad';
+export type { TextBoxExtractor } from './detectors';
+
+/**
+ * Configuration for the OCR pipeline: a model declares its input-size buckets, its
+ * charset, and its detector box-extraction strategy. The pipeline is
+ * architecture-agnostic — it validates the detect/recognize contract at load and
+ * takes everything model-specific here. The built-in {@link craftExtractBoxes} /
+ * {@link dbnetExtractBoxes} cover EasyOCR / PaddleOCR; other models supply their own
+ * {@link TextBoxExtractor} and override the recognizer normalization/padding/decode.
+ * @category Types
+ */
+export type OcrOptions = {
+ /**
+ * The model's static input-size buckets. The pipeline snaps each image to the
+ * closest `detect`/`recognize` bucket and calls the matching per-size method
+ * (`detect_` / `recognize_`). See {@link Buckets}.
+ */
+ readonly buckets: Buckets;
+ /**
+ * Recognizer charset (a string = one codepoint per index; an array is taken
+ * verbatim, for multi-codepoint entries like ligatures).
+ */
+ readonly charset: string | readonly string[];
+ /**
+ * Detector box-extraction strategy: maps the raw `detect_` outputs to oriented
+ * quads. Use the built-in {@link craftExtractBoxes} / {@link dbnetExtractBoxes}, or
+ * supply your own {@link TextBoxExtractor} to plug in a new detector.
+ */
+ readonly extractBoxes: TextBoxExtractor;
+ /** Drop detections scoring below this. Defaults to 0. */
+ readonly dropScore?: number;
+ /**
+ * Recognizer input normalization, applied after the warp as `x·alpha + beta`
+ * (scalar, or per-RGB-channel `[r,g,b]`). Defaults to `(x/255 − 0.5)/0.5` →
+ * `[−1,1]` (`alpha = 1/127.5`, `beta = −1`). Override for a recognizer trained
+ * with different normalization (e.g. ImageNet). The detector input norm is
+ * fixed by contract: RGB ÷ 255, with mean/std baked into the PTE.
+ */
+ readonly recognizerNorm?: {
+ readonly alpha: number | readonly number[];
+ readonly beta: number | readonly number[];
+ };
+ /** Recognizer canvas padding fill value. Defaults to 128 (neutral gray). */
+ readonly recognizerPadValue?: number;
+ /**
+ * Custom recognizer decode, replacing the built-in greedy CTC. Receives the raw
+ * `recognize_` output tensor (shape `[1, T, V]`) and the charset, and returns the
+ * recognized text plus a confidence in `[0,1]`. Use for non-CTC heads (attention/AR
+ * decoders) or custom scoring. MUST be a worklet.
+ */
+ readonly decode?: (
+ logits: Tensor,
+ charset: readonly string[]
+ ) => { readonly text: string; readonly confidence: number };
+};
+
+/**
+ * Per-run OCR options (passed to `runOcr`, not baked into the model — toggling
+ * them needs no reload).
+ * @category Types
+ */
+export type RunOcrOptions = {
+ /**
+ * Add handling for upright stacked columns (e.g. vertical signage, shipping-
+ * container codes — letters stacked top-to-bottom) on top of the normal
+ * horizontal read. X-aligned stacked glyph boxes are joined into one column
+ * word; a single tall box is cropped and its glyphs re-detected. Horizontal
+ * lines still read normally, so this only ADDS capability (at extra compute).
+ */
+ readonly vertical?: boolean;
+ /** Height/width ratio above which a box is treated as a stacked column. Default 1.5. */
+ readonly tallCropRatio?: number;
+ /** Max stacked-column re-detection passes per page (each is detector-scale). Default 8. */
+ readonly maxRedetections?: number;
+ /**
+ * Free the model's bucket-method activation arenas (`detect_`/`recognize_`)
+ * after this run, so memory doesn't accumulate as image/box sizes vary across
+ * runs (worse on CoreML, which compiles a graph per method). Default `true`.
+ * The document orchestrator passes `false` for its per-region OCR calls and
+ * frees once per page via `releaseMethodsWorklet` instead, so it keeps the run's
+ * working set cached while still bounding memory.
+ */
+ readonly release?: boolean;
+};
+
+/**
+ * Model configuration required to instantiate an OCR task runner: one fused PTE
+ * exposing the per-size `detect_` / `recognize_` methods, plus its options.
+ * @category Types
+ */
+export type OcrModel = {
+ readonly modelPath: string;
+ readonly ocrOpts: OcrOptions;
+};
+
+/**
+ * A single recognized text region.
+ * @category Types
+ */
+export type OcrDetection = {
+ readonly text: string;
+ readonly confidence: number;
+ /**
+ * The oriented quad (TL,TR,BR,BL) in original image pixels. Derive the
+ * axis-aligned bounds with `boundsOfPoints(quad, 'xyxy')` from `cv.ops.quad` if needed.
+ */
+ readonly quad: readonly Point[];
+};
+
+/**
+ * The result of one OCR run: the recognized text regions.
+ * @category Types
+ */
+export type OcrResult = {
+ readonly detections: OcrDetection[];
+};
+
+const RECOGNIZER_ALPHA = 1 / 127.5; // (x/255 - 0.5)/0.5 -> [-1, 1]
+const RECOGNIZER_BETA = -1;
+const RECOGNIZER_PAD_VALUE = 128; // neutral gray
+const TALL_CROP_RATIO = 1.5;
+const MAX_VERTICAL_REDETECTIONS = 8;
+// Vertical reads are lower-confidence and opt-in, so they skip the drop-score gate.
+const VERTICAL_DROP_SCORE = 0;
+
+function pushDetection(
+ out: OcrDetection[],
+ threshold: number,
+ text: string,
+ conf: number,
+ quad: readonly Point[]
+): void {
+ 'worklet';
+ if (text.length > 0 && conf >= threshold) {
+ out.push({ text, confidence: conf, quad });
+ }
+}
+
+/**
+ * Creates a unified OCR runner for two-stage detect → recognize models
+ * (EasyOCR / PaddleOCR). It loads one fused PTE, validates every
+ * `detect_` / `recognize_` bucket method, pre-allocates static scratch
+ * tensors sized from the model's compiled shapes, and returns recognition +
+ * disposal controls.
+ * @category Typescript API
+ * @param config The model path and OCR options ({@link OcrModel}).
+ * @param runtime Optional worklet runtime thread on which to run the pipeline.
+ * @returns A promise resolving to an object with recognition and disposal
+ * controls.
+ */
+export async function createOcr(
+ config: OcrModel,
+ runtime?: WorkletRuntime
+): Promise<{
+ dispose: () => void;
+ runOcr: (input: ImageBuffer, options?: RunOcrOptions) => Promise;
+ runOcrWorklet: (input: ImageBuffer, options?: RunOcrOptions) => OcrResult;
+ /** Free all bucket-method arenas without disposing the model (see `RunOcrOptions.release`). */
+ releaseMethodsWorklet: () => void;
+}> {
+ const { modelPath, ocrOpts } = config;
+ const model = await wrapAsync(loadModel, runtime)(modelPath);
+
+ const dropScore = ocrOpts.dropScore ?? 0;
+ const recNormAlpha = ocrOpts.recognizerNorm?.alpha ?? RECOGNIZER_ALPHA;
+ const recNormBeta = ocrOpts.recognizerNorm?.beta ?? RECOGNIZER_BETA;
+ const recPadValue = ocrOpts.recognizerPadValue ?? RECOGNIZER_PAD_VALUE;
+ const recDecode = ocrOpts.decode;
+
+ const detBuckets = ocrOpts.buckets.detect;
+ const recBuckets = ocrOpts.buckets.recognize;
+ // Validation + scratch allocation can throw; each tensor is pushed into
+ // `allocated` the moment it exists (one call per statement) so the catch can
+ // dispose every native allocation — a bad config must not leak.
+ const allocated: Tensor[] = [];
+ const recSets: RecSet[] = [];
+ let recC = 3;
+ let recH = 0;
+ let charset: string[] = [];
+ let recSetByWidth: ReadonlyMap = new Map();
+ const detSets: DetSet[] = [];
+ let detSetByS: ReadonlyMap = new Map();
+ try {
+ if (detBuckets.length === 0 || recBuckets.length === 0) {
+ throw new Error(
+ 'OCR: buckets.detect and buckets.recognize must each list at least one size.'
+ );
+ }
+ const detContract = resolveDetectorContract(model, detBuckets);
+ const rec = resolveRecognizerContract(model, recBuckets);
+ recC = rec.recC;
+ recH = rec.recH;
+ for (const bucket of rec.buckets) {
+ const tCanvas = tensor('uint8', [rec.recH, bucket.width, rec.recC]);
+ allocated.push(tCanvas);
+ const tCF = tensor('uint8', [rec.recC, rec.recH, bucket.width]);
+ allocated.push(tCF);
+ const tNorm = tensor('float32', [rec.recC, rec.recH, bucket.width]);
+ allocated.push(tNorm);
+ const tInput = tensor('float32', bucket.inShape);
+ allocated.push(tInput);
+ const tLogits = tensor('float32', bucket.outShape);
+ allocated.push(tLogits);
+ recSets.push({ width: bucket.width, tCanvas, tCF, tNorm, tInput, tLogits });
+ }
+ recSetByWidth = new Map(recSets.map((recSet) => [recSet.width, recSet]));
+
+ if (recC !== 3) {
+ throw new Error(`OCR: recognizer must take RGB (3 channels), but the model expects ${recC}.`);
+ }
+ // CTC lookup: index 0 is the blank (ctcCollapse never decodes it), then the
+ // model's characters — a string splits into codepoints, an array is taken
+ // verbatim (preserving multi-codepoint entries like ligatures).
+ charset = [
+ '[blank]',
+ ...(typeof ocrOpts.charset === 'string' ? Array.from(ocrOpts.charset) : ocrOpts.charset),
+ ];
+ if (charset.length !== rec.vocabSize) {
+ throw new Error(
+ `OCR: charset size (${charset.length}, incl. blank) must match recognizer output vocab (${rec.vocabSize}).`
+ );
+ }
+ for (const { s, outputs } of detContract) {
+ const tColor = tensor('uint8', [s, s, 3]);
+ allocated.push(tColor);
+ const tCF = tensor('uint8', [3, s, s]);
+ allocated.push(tCF);
+ const tNorm = tensor('float32', [3, s, s]);
+ allocated.push(tNorm);
+ const tInput = tensor('float32', [1, 3, s, s]);
+ allocated.push(tInput);
+ const tOutputs: Tensor[] = [];
+ for (const spec of outputs) {
+ const tOut = tensor(spec.dtype, spec.shape);
+ allocated.push(tOut);
+ tOutputs.push(tOut);
+ }
+ detSets.push({ s, tColor, tCF, tNorm, tInput, tOutputs });
+ }
+ detSetByS = new Map(detSets.map((detSet) => [detSet.s, detSet]));
+ } catch (e) {
+ for (const t of allocated) {
+ t.dispose();
+ }
+ model.dispose();
+ throw e;
+ }
+
+ const dispose = () => {
+ allocated.forEach((t) => t.dispose());
+ model.dispose();
+ };
+
+ // Frees each bucket method's activation arena without disposing the model; a
+ // freed method transparently reloads on its next execute. Must precede
+ // runOcrWorklet: the worklet plugin resolves referenced worklets by source order.
+ const releaseMethodsWorklet = () => {
+ 'worklet';
+ for (const s of detBuckets) {
+ model.unloadMethod(`detect_${s}`);
+ }
+ for (const w of recBuckets) {
+ model.unloadMethod(`recognize_${w}`);
+ }
+ };
+
+ const runOcrWorklet = (input: ImageBuffer, options?: RunOcrOptions): OcrResult => {
+ 'worklet';
+ const vertical = options?.vertical ?? false;
+ const tallCropRatio = options?.tallCropRatio ?? TALL_CROP_RATIO;
+ const maxRedetections = options?.maxRedetections ?? MAX_VERTICAL_REDETECTIONS;
+ const release = options?.release ?? true;
+ const { data, width, height, format } = input;
+ const numChannels = FORMAT_CHANNELS[format];
+ const rgbCode = FORMAT_CONVERSION[format].rgb;
+
+ const detCtx: DetectContext = {
+ model,
+ detBuckets,
+ numChannels,
+ toRgbCode: rgbCode,
+ extractBoxes: ocrOpts.extractBoxes,
+ detSets: detSetByS,
+ };
+
+ const tInputRaw = tensor('uint8', [height, width, numChannels]);
+ let tRecImage: Tensor | null = null;
+ try {
+ tInputRaw.setData(data);
+
+ const quads = detectQuads(detCtx, tInputRaw, width, height);
+ if (quads.length === 0) {
+ return { detections: [] };
+ }
+
+ let recSrc = tInputRaw;
+ if (rgbCode !== null) {
+ tRecImage = tensor('uint8', [height, width, recC]);
+ recSrc = cvtColor(tInputRaw, tRecImage, rgbCode);
+ }
+ const recCtx: RecContext = {
+ model,
+ recSetByWidth,
+ recBuckets,
+ recH,
+ charset,
+ normAlpha: recNormAlpha,
+ normBeta: recNormBeta,
+ padValue: recPadValue,
+ decode: recDecode,
+ };
+ const verticalCtx: VerticalContext = {
+ detCtx,
+ rawPage: tInputRaw,
+ recC,
+ tallCropRatio,
+ redetectBudget: { remaining: maxRedetections },
+ };
+
+ const detections: OcrDetection[] = [];
+
+ const ordered: Point[][] = [];
+ for (const quad of quads) {
+ const orderedQuad = orderQuad(quad);
+ const size = quadSize(orderedQuad);
+ if (size.width >= 3 && size.height >= 3) {
+ ordered.push(orderedQuad);
+ }
+ }
+
+ if (!vertical) {
+ for (const orderedQuad of ordered) {
+ const { text, conf } = recognizeQuad(recCtx, recSrc, orderedQuad);
+ pushDetection(detections, dropScore, text, conf, orderedQuad);
+ }
+ return { detections: orderByReadingOrder(detections) };
+ }
+
+ const { columns, singles } = groupVerticalColumns(ordered);
+ for (const col of columns) {
+ const strip = recognizeGlyphStrip(recCtx, recSrc, col);
+ if (strip) {
+ pushDetection(
+ detections,
+ VERTICAL_DROP_SCORE,
+ strip.text,
+ strip.conf,
+ boundingQuadOf(col)
+ );
+ }
+ }
+ for (const orderedQuad of singles) {
+ const size = quadSize(orderedQuad);
+ if (size.height >= size.width * verticalCtx.tallCropRatio) {
+ const stacked = readStackedColumn(recCtx, verticalCtx, orderedQuad, size);
+ if (stacked) {
+ pushDetection(detections, VERTICAL_DROP_SCORE, stacked.text, stacked.conf, orderedQuad);
+ continue;
+ }
+ }
+ const { text, conf } = recognizeQuad(recCtx, recSrc, orderedQuad);
+ pushDetection(detections, dropScore, text, conf, orderedQuad);
+ }
+ return { detections: orderByReadingOrder(detections) };
+ } finally {
+ tInputRaw.dispose();
+ tRecImage?.dispose();
+ if (release) {
+ releaseMethodsWorklet();
+ }
+ }
+ };
+
+ const runOcr = wrapAsync(runOcrWorklet, runtime);
+
+ return { runOcr, runOcrWorklet, dispose, releaseMethodsWorklet };
+}
diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/ocr/ocrUtils.ts b/packages/react-native-executorch/src/extensions/cv/tasks/ocr/ocrUtils.ts
new file mode 100644
index 0000000000..32c18131ad
--- /dev/null
+++ b/packages/react-native-executorch/src/extensions/cv/tasks/ocr/ocrUtils.ts
@@ -0,0 +1,284 @@
+import type { Point } from '../../ops/points';
+import { boundsOfPoints } from '../../ops/quad';
+
+/**
+ * The static input-size buckets a bucketed OCR model exposes. Each model ships
+ * per-size detect (`detect_`, square `S×S` input) and recognize
+ * (`recognize_`, fixed height, width `W`) methods; the pipeline snaps each
+ * input to the nearest bucket and calls the matching method. Both lists must be
+ * ascending.
+ * @category Types
+ */
+export type Buckets = {
+ readonly detect: readonly number[];
+ readonly recognize: readonly number[];
+};
+
+// A size within this fraction above the next-lower bucket snaps DOWN to it rather
+// than up, so a marginal overflow (e.g. 641 px against a 640 bucket) doesn't jump
+// to the next, much larger and slower, bucket. Kept small: a larger downscale
+// loses detail the model was trained to see.
+const BUCKET_SNAP_TOLERANCE = 0.1;
+
+/**
+ * Selects the smallest bucket that fits `size`, but snaps down to the next-lower
+ * bucket when `size` exceeds it by no more than the snap tolerance; oversized
+ * inputs clamp to the largest bucket. Detector callers pass the image's longest
+ * side (selecting `detect_`); recognizer callers pass the desired crop
+ * content width (selecting `recognize_`).
+ * @category Typescript API
+ * @param size The size to fit, in pixels.
+ * @param buckets The model's ascending bucket sizes.
+ * @returns The selected bucket size.
+ */
+export function snapBucket(size: number, buckets: readonly number[]): number {
+ 'worklet';
+ for (let i = 0; i < buckets.length; i++) {
+ if (buckets[i]! >= size) {
+ const lower = i > 0 ? buckets[i - 1]! : 0;
+ return lower > 0 && size <= lower * (1 + BUCKET_SNAP_TOLERANCE) ? lower : buckets[i]!;
+ }
+ }
+ return buckets[buckets.length - 1]!;
+}
+
+/**
+ * Computes the content width (px) of a recognizer crop: the region resized to the
+ * recognizer height keeping its aspect ratio, clamped to the bucket width.
+ * @category Typescript API
+ * @param regionWidth The region's natural width.
+ * @param regionHeight The region's natural height.
+ * @param recognizerHeight The recognizer input height.
+ * @param bucketWidth The recognizer input (canvas) width.
+ * @returns The clamped content width in pixels.
+ */
+export function contentWidthFor(
+ regionWidth: number,
+ regionHeight: number,
+ recognizerHeight: number,
+ bucketWidth: number
+): number {
+ 'worklet';
+ const width = Math.round((recognizerHeight * regionWidth) / Math.max(1, regionHeight));
+ return Math.max(1, Math.min(width, bucketWidth));
+}
+
+// A gutter must be at least this fraction of the content width to split columns;
+// two boxes share a line when their vertical extents overlap by at least this
+// fraction of the shorter box's height.
+const COLUMN_GAP_FRACTION = 0.06;
+const LINE_OVERLAP_FRACTION = 0.3;
+
+/**
+ * Reorders items carrying a `quad` into human reading order: multi-column inputs
+ * read column-by-column, single-column inputs line-by-line, and boxes within a
+ * line left-to-right. Detectors emit boxes in an arbitrary order, so detections
+ * and assembled blocks are ordered through this.
+ * @category Typescript API
+ * @param items The items to reorder, each carrying a `quad`.
+ * @returns The items in reading order.
+ */
+export function orderByReadingOrder(items: T[]): T[] {
+ 'worklet';
+ const count = items.length;
+ if (count <= 1) {
+ return items;
+ }
+
+ // 1. Axis-aligned bounds per quad, plus the content x-range: a column gutter
+ // must be at least COLUMN_GAP_FRACTION of that range to count.
+ const boxes = items.map((it) => boundsOfPoints(it.quad, 'xyxy'));
+ let minX = Infinity;
+ let maxX = -Infinity;
+ for (const box of boxes) {
+ if (box.xmin < minX) minX = box.xmin;
+ if (box.xmax > maxX) maxX = box.xmax;
+ }
+ const minGap = COLUMN_GAP_FRACTION * Math.max(1, maxX - minX);
+
+ // 2. Find column gutters with an x-coverage sweep over the box edges: while
+ // inside any box the coverage counter is > 0; a zero-coverage span wider
+ // than minGap is a gutter, and its midpoint becomes a column cut.
+ const edges: { x: number; delta: number }[] = [];
+ for (const box of boxes) {
+ edges.push({ x: box.xmin, delta: 1 });
+ edges.push({ x: box.xmax, delta: -1 });
+ }
+ // At equal x, open (+1) before close (-1) so touching boxes don't open a gutter.
+ edges.sort((a, b) => a.x - b.x || b.delta - a.delta);
+ const cuts: number[] = [];
+ let coverage = 0;
+ let gutterStart = 0;
+ for (const edge of edges) {
+ const before = coverage;
+ coverage += edge.delta;
+ if (before > 0 && coverage === 0) {
+ gutterStart = edge.x;
+ } else if (before === 0 && coverage > 0 && edge.x - gutterStart >= minGap) {
+ cuts.push((gutterStart + edge.x) / 2);
+ }
+ }
+
+ // 3. Assign each box to a column: count how many (ascending) cuts its
+ // center-x lies to the right of.
+ const columns: number[][] = Array.from({ length: cuts.length + 1 }, () => []);
+ for (let i = 0; i < count; i++) {
+ const centerX = (boxes[i]!.xmin + boxes[i]!.xmax) / 2;
+ let column = 0;
+ for (const cut of cuts) {
+ if (centerX > cut) column++;
+ }
+ columns[column]!.push(i);
+ }
+
+ // 4. Within each (left-to-right) column: group boxes into lines by vertical
+ // overlap (≥ LINE_OVERLAP_FRACTION of the shorter height joins a line),
+ // order lines top-to-bottom and boxes within a line left-to-right.
+ const order: number[] = [];
+ for (const column of columns) {
+ column.sort((a, b) => boxes[a]!.ymin - boxes[b]!.ymin);
+ const lines: { items: number[]; ymin: number; ymax: number }[] = [];
+ for (const i of column) {
+ const box = boxes[i]!;
+ let placed = false;
+ for (const line of lines) {
+ const overlap = Math.min(line.ymax, box.ymax) - Math.max(line.ymin, box.ymin);
+ const minHeight = Math.min(line.ymax - line.ymin, box.ymax - box.ymin);
+ if (overlap >= LINE_OVERLAP_FRACTION * Math.max(1, minHeight)) {
+ line.items.push(i);
+ line.ymin = Math.min(line.ymin, box.ymin);
+ line.ymax = Math.max(line.ymax, box.ymax);
+ placed = true;
+ break;
+ }
+ }
+ if (!placed) {
+ lines.push({ items: [i], ymin: box.ymin, ymax: box.ymax });
+ }
+ }
+ lines.sort((a, b) => a.ymin - b.ymin);
+ for (const line of lines) {
+ line.items.sort(
+ (a, b) => boxes[a]!.xmin + boxes[a]!.xmax - (boxes[b]!.xmin + boxes[b]!.xmax)
+ );
+ order.push(...line.items);
+ }
+ }
+ return order.map((i) => items[i]!);
+}
+
+// A box wider than this multiple of its height is a horizontal line, never a
+// stacked-column glyph. A box joins a column when its x-span overlaps the
+// column's by COLUMN_X_OVERLAP of the narrower width and the y-gap is within
+// COLUMN_Y_GAP of its height.
+const COLUMN_GLYPH_ASPECT = 1.6;
+const COLUMN_X_OVERLAP = 0.25;
+const COLUMN_Y_GAP = 2.5;
+
+/**
+ * Clusters glyph-like, x-aligned, vertically-stacked boxes into columns; wide
+ * lines and isolated boxes are returned as `singles` to read normally. This lets
+ * a vertical-text pass add column reading without disturbing horizontal reads.
+ * @category Typescript API
+ * @param quads The detected text quads (ordered TL,TR,BR,BL).
+ * @returns The detected `columns` (each a top-to-bottom list of quads) and the
+ * leftover `singles` (horizontal lines / isolated boxes).
+ */
+export function groupVerticalColumns(quads: readonly (readonly Point[])[]): {
+ columns: Point[][][];
+ singles: Point[][];
+} {
+ 'worklet';
+ type Candidate = {
+ quad: Point[];
+ xmin: number;
+ xmax: number;
+ ymin: number;
+ ymax: number;
+ width: number;
+ height: number;
+ };
+ const candidates: Candidate[] = [];
+ const singles: Point[][] = [];
+ for (const q of quads) {
+ const { xmin, ymin, xmax, ymax } = boundsOfPoints(q, 'xyxy');
+ const width = xmax - xmin;
+ const height = ymax - ymin;
+ if (width > height * COLUMN_GLYPH_ASPECT) {
+ singles.push(q as Point[]);
+ } else {
+ candidates.push({ quad: q as Point[], xmin, xmax, ymin, ymax, width, height });
+ }
+ }
+ // Grow each column top-to-bottom from its current bottom box, checking alignment
+ // against the column's accumulated x-range so a narrow glyph between wider ones
+ // doesn't break the run.
+ candidates.sort((a, b) => a.ymin - b.ymin);
+ type Column = { boxes: Candidate[]; xmin: number; xmax: number; bottom: number };
+ const columns: Column[] = [];
+ for (const box of candidates) {
+ let placed = false;
+ for (const column of columns) {
+ const overlap = Math.min(box.xmax, column.xmax) - Math.max(box.xmin, column.xmin);
+ const aligned = overlap > COLUMN_X_OVERLAP * Math.min(box.width, column.xmax - column.xmin);
+ const gap = box.ymin - column.bottom;
+ if (aligned && gap < COLUMN_Y_GAP * box.height && gap > -0.5 * box.height) {
+ column.boxes.push(box);
+ column.xmin = Math.min(column.xmin, box.xmin);
+ column.xmax = Math.max(column.xmax, box.xmax);
+ column.bottom = box.ymax;
+ placed = true;
+ break;
+ }
+ }
+ if (!placed) {
+ columns.push({ boxes: [box], xmin: box.xmin, xmax: box.xmax, bottom: box.ymax });
+ }
+ }
+ const grouped: Point[][][] = [];
+ for (const column of columns) {
+ if (column.boxes.length >= 2) {
+ grouped.push(column.boxes.map((b) => b.quad));
+ } else {
+ singles.push(column.boxes[0]!.quad);
+ }
+ }
+ return { columns: grouped, singles };
+}
+
+/**
+ * Collapses a greedy-CTC argmax path into recognized text and a confidence score:
+ * reserved low indices (the CTC blank, `< numSpecials`) and consecutive repeats
+ * are dropped from the text, and the confidence is the mean value over the
+ * non-reserved timesteps.
+ * @category Typescript API
+ * @param indices The per-timestep argmax indices.
+ * @param values The per-timestep max values (probabilities).
+ * @param charset The charset lookup, reserved tokens at the front.
+ * @param numSpecials Number of reserved low indices (default 1 = CTC blank).
+ * @returns The decoded `text` and its `confidence` in `[0, 1]`.
+ */
+export function ctcCollapse(
+ indices: number[],
+ values: number[],
+ charset: readonly string[],
+ numSpecials = 1
+): { text: string; confidence: number } {
+ 'worklet';
+ let text = '';
+ let last = -1;
+ let sum = 0;
+ let count = 0;
+ for (let i = 0; i < indices.length; i++) {
+ const idx = indices[i]!;
+ if (idx >= numSpecials) {
+ sum += values[i]!;
+ count++;
+ if (idx !== last && idx < charset.length) {
+ text += charset[idx]!;
+ }
+ }
+ last = idx;
+ }
+ return { text, confidence: count === 0 ? 0 : sum / count };
+}
diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/ocr/pipeline.ts b/packages/react-native-executorch/src/extensions/cv/tasks/ocr/pipeline.ts
new file mode 100644
index 0000000000..5397b41dd0
--- /dev/null
+++ b/packages/react-native-executorch/src/extensions/cv/tasks/ocr/pipeline.ts
@@ -0,0 +1,347 @@
+// OCR pipeline engine: the per-page / per-box worklet functions and the
+// construction-time contract resolvers behind `createOcr` (in ocr.ts). Internal
+// to the OCR task — not re-exported from the package index.
+//
+// Worklet source-order: the worklet plugin captures a referenced worklet by its
+// position in this file, so a worklet must be defined before any worklet that
+// calls it (recognizeCanvas -> recognizeQuad/recognizeGlyphStrip;
+// recognizeGlyphStrip -> readStackedColumn). The construction-time resolvers run
+// on the JS thread and have no such constraint.
+
+import { rnexecutorchJsi } from '../../../../native/bridge';
+import { tensor, type Tensor, type DType } from '../../../../core/tensor';
+import { validateModelSchema, SymbolicTensor } from '../../../../core/modelSchema';
+import type { Model } from '../../../../core/model';
+
+import type { Point } from '../../ops/points';
+import {
+ resize,
+ cvtColor,
+ toChannelsFirst,
+ normalize,
+ warpQuad,
+ type ColorConversionCode,
+} from '../../ops/image';
+import { mapQuadToImage, orderQuad, quadSize, flattenQuad, splitTallQuad } from '../../ops/quad';
+import type { TextBoxExtractor } from './detectors';
+import { contentWidthFor, ctcCollapse, snapBucket } from './ocrUtils';
+
+// The detector consumes raw RGB scaled to [0,1]; its mean/std normalization is
+// baked into the model, so the client only divides by 255.
+const DETECTOR_ALPHA = 1 / 255;
+const DETECTOR_BETA = 0;
+
+// Per-timestep argmax + max value over recognizer logits `[..,T,V]`, computed
+// natively on the tensor buffer (avoids copying the whole tensor into JS). The
+// native op returns a flat [idx, value, ...] array, reshaped here.
+function ctcGreedyDecode(src: Tensor): { indices: number[]; values: number[] } {
+ 'worklet';
+ const flat = rnexecutorchJsi.cv.ctcGreedyDecode(src) as number[];
+ const indices: number[] = [];
+ const values: number[] = [];
+ for (let i = 0; i < flat.length; i += 2) {
+ indices.push(flat[i]!);
+ values.push(flat[i + 1]!);
+ }
+ return { indices, values };
+}
+
+/**
+ * Per-detect-bucket scratch tensors, allocated once and reused across runs:
+ * `tColor [s,s,3]` → `tCF [3,s,s]` → `tNorm [3,s,s]` → `tInput [1,3,s,s]`.
+ */
+export type DetSet = {
+ readonly s: number;
+ readonly tColor: Tensor;
+ readonly tCF: Tensor;
+ readonly tNorm: Tensor;
+ readonly tInput: Tensor;
+ readonly tOutputs: readonly Tensor[];
+};
+
+/** The detector state, bundled so it can run on the full page or on a box crop. */
+export type DetectContext = {
+ readonly model: Model;
+ readonly detBuckets: readonly number[];
+ readonly numChannels: number;
+ readonly toRgbCode: ColorConversionCode | null;
+ readonly extractBoxes: TextBoxExtractor;
+ readonly detSets: ReadonlyMap;
+};
+
+/** Per-recognize-width scratch tensors, allocated once and reused across runs. */
+export type RecSet = {
+ readonly width: number;
+ readonly tCanvas: Tensor;
+ readonly tCF: Tensor;
+ readonly tNorm: Tensor;
+ readonly tInput: Tensor;
+ readonly tLogits: Tensor;
+};
+
+/** The recognizer state; the source image is passed per call. */
+export type RecContext = {
+ readonly model: Model;
+ readonly recSetByWidth: ReadonlyMap;
+ readonly recBuckets: readonly number[];
+ readonly recH: number;
+ readonly charset: string[];
+ readonly normAlpha: number | readonly number[];
+ readonly normBeta: number | readonly number[];
+ readonly padValue: number;
+ /** Optional custom decode; falls back to greedy CTC when absent. */
+ readonly decode?: (
+ logits: Tensor,
+ charset: readonly string[]
+ ) => { readonly text: string; readonly confidence: number };
+};
+
+/** Extra state the vertical-text path needs: the detector and the source page. */
+export type VerticalContext = {
+ readonly detCtx: DetectContext;
+ readonly rawPage: Tensor;
+ readonly recC: number;
+ /** Height/width ratio above which a box is treated as a stacked column. */
+ readonly tallCropRatio: number;
+ /** Per-page budget for the (expensive) stacked-column re-detection pass. */
+ readonly redetectBudget: { remaining: number };
+};
+
+// Detects text boxes in `src` and returns their quads in `src` pixel space:
+// letterbox into the snapped square bucket, run the detector, and hand the raw
+// outputs to the model's extractor. `charLevel` requests per-glyph boxes for the
+// stacked-text pass.
+export function detectQuads(
+ ctx: DetectContext,
+ src: Tensor,
+ width: number,
+ height: number,
+ charLevel = false
+): Point[][] {
+ 'worklet';
+ const detS = snapBucket(Math.max(width, height), ctx.detBuckets);
+ const detSet = ctx.detSets.get(detS)!;
+ // Only the source resize depends on the run's channel count; the rest is cached.
+ const tDetResize = tensor('uint8', [detS, detS, ctx.numChannels]);
+ try {
+ src
+ .through(resize, tDetResize, { mode: 'letterbox', interpolation: 'area', padValue: 0 })
+ .throughIf(ctx.toRgbCode !== null, cvtColor, detSet.tColor, ctx.toRgbCode!)
+ .through(toChannelsFirst, detSet.tCF)
+ .through(normalize, detSet.tNorm, { alpha: DETECTOR_ALPHA, beta: DETECTOR_BETA })
+ .copyTo(detSet.tInput);
+
+ ctx.model.execute(`detect_${detS}`, [detSet.tInput], [...detSet.tOutputs]);
+ const quads = ctx.extractBoxes(detSet.tOutputs, detS, charLevel);
+ return quads.map((q) => mapQuadToImage(q, detS, detS, width, height));
+ } finally {
+ tDetResize.dispose();
+ }
+}
+
+// Normalizes the already-warped recognizer canvas, runs the recognizer, and
+// decodes the logits — a custom decode if the model provides one, else greedy CTC
+// (the recognizer emits probabilities). Callers prepare `tCanvas` via `warpQuad`.
+function recognizeCanvas(
+ recCtx: RecContext,
+ recSet: RecSet,
+ bucketW: number
+): { text: string; conf: number } {
+ 'worklet';
+ recSet.tCanvas
+ .through(toChannelsFirst, recSet.tCF)
+ .through(normalize, recSet.tNorm, { alpha: recCtx.normAlpha, beta: recCtx.normBeta })
+ .copyTo(recSet.tInput);
+ recCtx.model.execute(`recognize_${bucketW}`, [recSet.tInput], [recSet.tLogits]);
+ if (recCtx.decode) {
+ const r = recCtx.decode(recSet.tLogits, recCtx.charset);
+ return { text: r.text, conf: r.confidence };
+ }
+ const { indices, values } = ctcGreedyDecode(recSet.tLogits);
+ const { text, confidence } = ctcCollapse(indices, values, recCtx.charset);
+ return { text, conf: confidence };
+}
+
+// Recognizes one ordered (TL,TR,BR,BL) quad from `src`: snap its content width to
+// a recognizer bucket, warp it into the canvas, then recognize.
+export function recognizeQuad(
+ ctx: RecContext,
+ src: Tensor,
+ corners: readonly Point[]
+): { text: string; conf: number } {
+ 'worklet';
+ const size = quadSize(corners);
+ const maxRec = ctx.recBuckets[ctx.recBuckets.length - 1]!;
+ const desiredW = contentWidthFor(size.width, size.height, ctx.recH, maxRec);
+ const bucketW = snapBucket(desiredW, ctx.recBuckets);
+ const recSet = ctx.recSetByWidth.get(bucketW)!;
+ warpQuad(src, recSet.tCanvas, flattenQuad(corners), {
+ contentWidth: Math.min(desiredW, bucketW),
+ align: 'left',
+ padMode: 'constant',
+ padValue: ctx.padValue,
+ });
+ return recognizeCanvas(ctx, recSet, bucketW);
+}
+
+// Recognizes a sequence of glyph quads as a single line: each glyph is warped
+// upright to the recognizer height and placed side by side in one canvas (the
+// native warp composes directly, so there is no JS pixel assembly), then read in
+// one pass. A glyph box much taller than wide is first split into ~square
+// single-letter cells. Returns null when no usable text was produced.
+export function recognizeGlyphStrip(
+ recCtx: RecContext,
+ src: Tensor,
+ glyphs: readonly (readonly Point[])[]
+): { text: string; conf: number } | null {
+ 'worklet';
+ const recH = recCtx.recH;
+ const maxRec = recCtx.recBuckets[recCtx.recBuckets.length - 1]!;
+ // Split any tall multi-letter box into single-letter cells and size each cell's
+ // warped width (aspect preserved) to lay out the strip.
+ const cells: { quad: readonly Point[]; width: number }[] = [];
+ let totalW = 0;
+ for (const glyph of glyphs) {
+ const glyphSize = quadSize(glyph);
+ if (glyphSize.width < 1 || glyphSize.height < 1) {
+ continue;
+ }
+ const parts = Math.max(1, Math.round(glyphSize.height / Math.max(1, glyphSize.width)));
+ for (const cell of splitTallQuad(glyph, parts)) {
+ const cellSize = quadSize(cell);
+ if (cellSize.width < 1 || cellSize.height < 1) {
+ continue;
+ }
+ const width = contentWidthFor(cellSize.width, cellSize.height, recH, maxRec);
+ cells.push({ quad: cell, width });
+ totalW += width;
+ }
+ }
+ if (cells.length === 0) {
+ return null;
+ }
+ const bucketW = snapBucket(totalW, recCtx.recBuckets);
+ const recSet = recCtx.recSetByWidth.get(bucketW)!;
+ // Warp each cell into the canvas at its x-offset; the first warp clears + pads
+ // the whole canvas, the rest compose in with `clear: false`.
+ let xOff = 0;
+ for (let i = 0; i < cells.length; i++) {
+ if (xOff >= bucketW) {
+ break;
+ }
+ warpQuad(src, recSet.tCanvas, flattenQuad(cells[i]!.quad), {
+ contentWidth: cells[i]!.width,
+ offsetX: xOff,
+ clear: i === 0,
+ padMode: 'constant',
+ padValue: recCtx.padValue,
+ });
+ xOff += cells[i]!.width;
+ }
+ const { text, conf } = recognizeCanvas(recCtx, recSet, bucketW);
+ return text.length > 0 ? { text, conf } : null;
+}
+
+// Reads one tall box that the detector merged from several vertically-stacked
+// glyphs: crops it upright, re-detects the individual glyphs (char-level pass),
+// and recognizes them top-to-bottom via `recognizeGlyphStrip`. Returns null — the
+// caller then reads the box horizontally — when the box is too small, the per-page
+// re-detect budget is spent, or no glyphs are found.
+export function readStackedColumn(
+ recCtx: RecContext,
+ verticalCtx: VerticalContext,
+ ordered: readonly Point[],
+ size: { width: number; height: number }
+): { text: string; conf: number } | null {
+ 'worklet';
+ const boxW = Math.round(size.width);
+ const boxH = Math.round(size.height);
+ if (boxW < 3 || boxH < 3 || verticalCtx.redetectBudget.remaining <= 0) {
+ return null;
+ }
+ verticalCtx.redetectBudget.remaining--;
+ const tBoxRaw = tensor('uint8', [boxH, boxW, verticalCtx.detCtx.numChannels]);
+ // RGB conversion target — allocated lazily, only when the crop isn't RGB.
+ let tRecBox: Tensor | null = null;
+ try {
+ warpQuad(verticalCtx.rawPage, tBoxRaw, flattenQuad(ordered), {
+ contentWidth: boxW,
+ align: 'left',
+ padMode: 'constant',
+ padValue: 0,
+ });
+ const charQuads = detectQuads(verticalCtx.detCtx, tBoxRaw, boxW, boxH, /* charLevel */ true);
+ if (charQuads.length === 0) {
+ return null;
+ }
+ let boxSrc = tBoxRaw;
+ if (verticalCtx.detCtx.toRgbCode !== null) {
+ tRecBox = tensor('uint8', [boxH, boxW, verticalCtx.recC]);
+ boxSrc = cvtColor(tBoxRaw, tRecBox, verticalCtx.detCtx.toRgbCode);
+ }
+ // Read the stack top-to-bottom by each glyph's upper edge.
+ const glyphs = charQuads.map((q) => orderQuad(q)).sort((a, b) => a[0]!.y - b[0]!.y);
+ return recognizeGlyphStrip(recCtx, boxSrc, glyphs);
+ } finally {
+ tBoxRaw.dispose();
+ tRecBox?.dispose();
+ }
+}
+
+// Validates each `detect_` method against the shared RGB `[1, 3, S, S]` input
+// contract and reads its (model-defined) output tensor specs from the model
+// metadata. Returns specs only — the task factory allocates and owns the tensors.
+// Runs at construction; throws on a contract mismatch.
+export function resolveDetectorContract(
+ model: Model,
+ detBuckets: readonly number[]
+): { s: number; outputs: { dtype: DType; shape: number[] }[] }[] {
+ return detBuckets.map((s) => {
+ const method = `detect_${s}`;
+ // Match the declared output count with wildcard specs so validateModelSchema
+ // enforces the RGB input contract without constraining the model's outputs.
+ const outCount = model.getMethodMeta(method).outputTensorMeta.length;
+ const meta = validateModelSchema(
+ model,
+ method,
+ [SymbolicTensor('float32', [1, 3, s, s])],
+ Array.from({ length: outCount }, () => SymbolicTensor())
+ );
+ return { s, outputs: meta.outputTensorMeta.map((m) => ({ dtype: m.dtype, shape: m.shape })) };
+ });
+}
+
+// Validates each `recognize_` method and reads the recognizer contract: the
+// channel/height/vocab dimensions (constant across widths) plus each bucket's
+// input/output shapes. Returns specs only — the task factory allocates the
+// tensors. Runs at construction; throws on a contract mismatch.
+export function resolveRecognizerContract(
+ model: Model,
+ recBuckets: readonly number[]
+): {
+ recC: number;
+ recH: number;
+ vocabSize: number;
+ buckets: { width: number; inShape: number[]; outShape: number[] }[];
+} {
+ let recC = 0;
+ let recH = 0;
+ let vocabSize = 0;
+ const buckets = recBuckets.map((w, i) => {
+ const m = validateModelSchema(
+ model,
+ `recognize_${w}`,
+ [SymbolicTensor('float32', [1, 'C', 'H', 'W'])],
+ [SymbolicTensor('float32', [1, 'T', 'V'])]
+ );
+ const inShape = m.inputTensorMeta[0]!.shape;
+ const outShape = m.outputTensorMeta[0]!.shape;
+ if (i === 0) {
+ recC = inShape[1]!;
+ recH = inShape[2]!;
+ vocabSize = outShape[2]!;
+ }
+ return { width: w, inShape, outShape };
+ });
+ return { recC, recH, vocabSize, buckets };
+}
diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/preprocessing.ts b/packages/react-native-executorch/src/extensions/cv/tasks/preprocessing.ts
index 3569210481..202fe71999 100644
--- a/packages/react-native-executorch/src/extensions/cv/tasks/preprocessing.ts
+++ b/packages/react-native-executorch/src/extensions/cv/tasks/preprocessing.ts
@@ -1,7 +1,7 @@
import { tensor, type Tensor } from '../../../core/tensor';
import { matchShape } from '../../../core/modelSchema';
-import type { ImageBuffer } from '../image';
+import type { ImageBuffer, ImageFormat } from '../image';
import {
type ResizeMode,
type InterpolationMethod,
@@ -56,6 +56,16 @@ export function createImagePreprocessor(
* data.
*/
process: (input: ImageBuffer) => Tensor;
+ /**
+ * Like {@link process}, but reads from a full-res image tensor (`[H, W, C]`,
+ * uint8) already on-device instead of an `ImageBuffer`, avoiding the raw-data
+ * copy. `format` supplies the source channel count and color conversion. The
+ * returned tensor is preprocessor-managed (do not dispose).
+ * @param src The full-res source image tensor in HWC layout.
+ * @param format The pixel format of `src` (for channels + color conversion).
+ * @returns A reference to the managed output tensor.
+ */
+ processTensor: (src: Tensor, format: ImageFormat) => Tensor;
/**
* Releases all allocated native resources.
*/
@@ -114,5 +124,27 @@ export function createImagePreprocessor(
return tOutput;
};
- return { process, dispose };
+ const processTensor = (src: Tensor, format: ImageFormat): Tensor => {
+ 'worklet';
+ const numChannels = FORMAT_CHANNELS[format];
+ const colorCode = FORMAT_CONVERSION[format].rgb;
+ const tResize = tensor('uint8', [targetH, targetW, numChannels]);
+ try {
+ src
+ .through(resize, tResize, {
+ mode: resizeMode,
+ interpolation: interpolation,
+ padValue: padValue,
+ })
+ .throughIf(colorCode !== null, cvtColor, tColor, colorCode!)
+ .through(toChannelsFirst, tChanFirst)
+ .through(normalize, tNorm, { alpha, beta })
+ .copyTo(tOutput);
+ } finally {
+ tResize.dispose();
+ }
+ return tOutput;
+ };
+
+ return { process, processTensor, dispose };
}
diff --git a/packages/react-native-executorch/src/hooks/useDocumentOcr.ts b/packages/react-native-executorch/src/hooks/useDocumentOcr.ts
new file mode 100644
index 0000000000..810e2ff4cd
--- /dev/null
+++ b/packages/react-native-executorch/src/hooks/useDocumentOcr.ts
@@ -0,0 +1,64 @@
+import { useModel } from './useModel';
+import { useResourceDownload } from './useResourceDownload';
+import { createDocumentOcr, type DocumentOcrModel } from '../extensions/cv/tasks/ocr/documentOcr';
+
+// Swap a model spec's hosted `modelPath` for its downloaded local path. Returns
+// undefined when the spec is absent (an optional model) or its path hasn't
+// finished downloading yet.
+const localize = (
+ spec: M | undefined,
+ localPath: string | undefined
+): M | undefined => (spec && localPath ? { ...spec, modelPath: localPath } : undefined);
+
+/**
+ * React hook for the document OCR pipeline: OCR + optional layout detection +
+ * optional document models (orientation/dewarp/table), assembled into reading-ordered
+ * blocks. Downloads/compiles all enabled models, tracks progress and errors, and
+ * cleans up native memory on unmount or config change.
+ * @category Hooks
+ * @typeParam L The type representing the layout region class labels.
+ * @param config OCR model + optional layout/document models + flags.
+ * @param options Hook options.
+ * @param options.preventLoad If true, prevents downloading and compiling the models.
+ * @returns Loading state, error, download progress, and the document run functions.
+ */
+export function useDocumentOcr(
+ config: DocumentOcrModel,
+ options?: { preventLoad?: boolean }
+) {
+ const ocrDl = useResourceDownload(config.ocr.modelPath, options?.preventLoad);
+ const layoutDl = useResourceDownload(config.layout?.modelPath, options?.preventLoad);
+ const documentModelsDl = useResourceDownload(
+ config.documentModels?.modelPath,
+ options?.preventLoad
+ );
+
+ const ocr = localize(config.ocr, ocrDl.localPath);
+ const layout = localize(config.layout, layoutDl.localPath);
+ const documentModels = localize(config.documentModels, documentModelsDl.localPath);
+ const ready =
+ !!ocr && (!config.layout || !!layout) && (!config.documentModels || !!documentModels);
+ const localConfig: DocumentOcrModel | null = ready
+ ? { ...config, ocr: ocr!, layout, documentModels }
+ : null;
+
+ const { model, error } = useModel(createDocumentOcr |