chromium/src/third_party/blink/public/mojom/handwriting/handwriting.mojom

// Copyright 2021 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

module handwriting.mojom;

// https://github.com/WICG/handwriting-recognition/blob/main/explainer.md

import "mojo/public/mojom/base/time.mojom";
import "ui/gfx/geometry/mojom/geometry.mojom";

// Represents a single point in a handwriting stroke.
// Corresponds to handwriting_point.idl.
struct HandwritingPoint {
  // Represent the horizontal (location.x) and vertical (location.y) location
  // of the point.
  // The top-left corner coordinate is (location.x=0, location.y=0).
  gfx.mojom.PointF location;
  // The time elapsed since the starting time (e.g. when the first ink point
  // of the drawing is captured).
  mojo_base.mojom.TimeDelta t;
};

// Represents a stroke which is just a series of points.
// Corresponds to handwriting_stroke.idl.
struct HandwritingStroke {
  array<HandwritingPoint> points;
};

// Represents a segment of a handwriting stroke in the grapheme detected.
// One `HandwritingDrawingSegment` can only refer one stroke, denoted by
// `stroke_index` which is the index of the stroke in the input stroke arrays
// (i.e., the first parameter of the `HandwritingRecognizer::GetPrediction`
// function).
// The reason we need this struct is that different parts of one single stroke
// can belong to different grapheme detected.
// Corresponds to handwriting_drawing_segment.idl.
struct HandwritingDrawingSegment {
  // The index of the corresponding stroke in the input stroke array.
  uint32 stroke_index;
  // The index of the first point in the stroke that belongs to this drawing
  // segment.
  uint32 begin_point_index;
  // The index of the last point in the stroke that belongs to this drawing
  // segment.
  uint32 end_point_index;
};

// Represents a segment detected.
// Corresponds to handwriting_segment.idl.
struct HandwritingSegment {
  // The string representation of this grapheme.
  string grapheme;
  // HandwritingPrediction.text.slice(begin_index, end_index) === grapheme
  // If the grapheme spans multiple Unicode code points,
  // `end_index - begin_index` is greater than 1.
  uint32 begin_index;
  uint32 end_index;
  array<HandwritingDrawingSegment> drawing_segments;
};

// Represents one single prediction result.
// The final prediction output is an array of it.
// Corresponds to handwriting_prediction.idl.
struct HandwritingPrediction {
  string text;
  array<HandwritingSegment> segmentation_result;
};

// Represents the hints provided to the recognizer for better performance.
// Corresponds to handwriting_hints.idl.
struct HandwritingHints {
  // The type of content to be recognized. The recognizer may use these to
  // better rank the recognition results. (e.g. "text", "email", "number",
  // "per-character").
  string recognition_type;
  // Identifies how the strokes are captured. (e.g. "touch", "mouse", "pen")
  string input_type;
  // The text that comes before the handwriting. This can be texts that were
  // previously recognized, or were given as the writing context (e.g.
  // "Write your name here:"). This is the linguistic context to help
  // disambiguate the handwriting (e.g. “Hello world” vs. “Hello word”).
  string text_context;
  // The maximum number of alternative predictions to generate.
  uint32 alternatives;
};

// Represents the struct for query features supported by the backend handwriting
// recognizer.
// Corresponds to handwriting_feature_query.idl.
struct HandwritingFeatureQuery {
  // If non-empty, queries whether the languages are supported.
  // Languages are IETF BCP 47 language tags, e.g., "en", "zh-CN", "zh-Hans".
  array<string> languages;
  // Whether the "alternative" feature is queried.
  bool alternatives;
  // Whether the "segmentation" feature is queried.
  bool segmentation_result;
};

// Because "bool" is not nullable, we need to define an enum to present the
// status. And this will also be helpful when we need to extend it, for
// example, adding `kNeedDownload`.
// Add `kNotQueried` to make it possible to use `mojo::Converter` to convert
// `HandwritingFeatureQueryResult` types (otherwise, there is no way to know
// whether a feature is queried or not in the converter function.
enum HandwritingFeatureStatus {
  kNotQueried,
  kSupported,
  kNotSupported,
};

// The feature query result.
// Corresponds to handwriting_feature_query_result.idl.
struct HandwritingFeatureQueryResult {
  HandwritingFeatureStatus languages = kNotQueried;
  HandwritingFeatureStatus alternatives = kNotQueried;
  HandwritingFeatureStatus segmentation_result = kNotQueried;
};

// Used in creating recognizer.
// Corresponds to handwriting_model_constraint.idl.
struct HandwritingModelConstraint {
  // Languages are IETF BCP 47 language tags, e.g., "en", "zh-CN", "zh-Hans".
  array<string> languages;
};

// Interface for a renderer to use a specific speech recognition backend.
// The browser handles the requests and forwards them to the appropriate
// backend.
interface HandwritingRecognizer {
  // Does the recognition and outputs the prediction result.
  // This is used by IDL API `blink::HandwritingDrawing::getPrediction`.
  // The input `strokes` and `hints` should both come from
  // `blink::HandwritingDrawing`.
  // If the returned `Optional` has no value, it means there is some error in
  // recognition. If the returned `Optional` has value but the array is empty,
  // it means the recognizer can not recognize anything from the input.
  GetPrediction(array<HandwritingStroke> strokes, HandwritingHints hints)
    => (array<HandwritingPrediction>? prediction);
};

// Per-document interface that allows the renderer to ask the browser for
// a specific speech recognizer backend, query capabilities, et cetera.
// Corresponds to `navigator_handwriting_recognition_service.idl`.
interface HandwritingRecognitionService {
  // Creates a recognizer.
  // This is used by IDL API
  // `blink::HandwritingRecognitionService::createHandwritingRecognizer`.
  CreateHandwritingRecognizer(HandwritingModelConstraint constraint)
    => (CreateHandwritingRecognizerResult result,
        pending_remote<HandwritingRecognizer>? handwriting_recognizer);
  // Queries the features supported by the platform.
  // This is used by IDL API
  // `blink::HandwritingRecognitionService::queryHandwritingRecognizerSupport`.
  QueryHandwritingRecognizerSupport(HandwritingFeatureQuery query)
    => (HandwritingFeatureQueryResult? result);
};

enum CreateHandwritingRecognizerResult {
  kOk,
  kError,
};