blob: 3a788d7ecd23c973047d19c84abd63205967d03e [file] [log] [blame]
// Copyright 2021 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
module handwriting.mojom;
// https://github.com/WICG/handwriting-recognition/blob/main/explainer.md
import "mojo/public/mojom/base/time.mojom";
import "ui/gfx/geometry/mojom/geometry.mojom";
// Represents a single point in a handwriting stroke.
// Corresponds to handwriting_point.idl.
struct HandwritingPoint {
// Represent the horizontal (location.x) and vertical (location.y) location
// of the point.
// The top-left corner coordinate is (location.x=0, location.y=0).
gfx.mojom.PointF location;
// The time elapsed since the starting time (e.g. when the first ink point
// of the drawing is captured).
mojo_base.mojom.TimeDelta t;
};
// Represents a stroke which is just a series of points.
// Corresponds to handwriting_stroke.idl.
struct HandwritingStroke {
array<HandwritingPoint> points;
};
// Represents a segment of a handwriting stroke in the grapheme detected.
// One `HandwritingDrawingSegment` can only refer one stroke, denoted by
// `stroke_index` which is the index of the stroke in the input stroke arrays
// (i.e., the first parameter of the `HandwritingRecognizer::GetPrediction`
// function).
// The reason we need this struct is that different parts of one single stroke
// can belong to different grapheme detected.
// Corresponds to handwriting_drawing_segment.idl.
struct HandwritingDrawingSegment {
// The index of the corresponding stroke in the input stroke array.
uint32 stroke_index;
// The index of the first point in the stroke that belongs to this drawing
// segment.
uint32 begin_point_index;
// The index of the last point in the stroke that belongs to this drawing
// segment.
uint32 end_point_index;
};
// Represents a segment detected.
// Corresponds to handwriting_segment.idl.
struct HandwritingSegment {
// The string representation of this grapheme.
string grapheme;
// HandwritingPrediction.text.slice(begin_index, end_index) === grapheme
// If the grapheme spans multiple Unicode code points,
// `end_index - begin_index` is greater than 1.
uint32 begin_index;
uint32 end_index;
array<HandwritingDrawingSegment> drawing_segments;
};
// Represents one single prediction result.
// The final prediction output is an array of it.
// Corresponds to handwriting_prediction.idl.
struct HandwritingPrediction {
string text;
array<HandwritingSegment> segmentation_result;
};
// Represents the hints provided to the recognizer for better performance.
// Corresponds to handwriting_hints.idl.
struct HandwritingHints {
// The type of content to be recognized. The recognizer may use these to
// better rank the recognition results. (e.g. "text", "email", "number",
// "per-character").
string recognition_type;
// Identifies how the strokes are captured. (e.g. "touch", "mouse", "pen")
string input_type;
// The text that comes before the handwriting. This can be texts that were
// previously recognized, or were given as the writing context (e.g.
// "Write your name here:"). This is the linguistic context to help
// disambiguate the handwriting (e.g. “Hello world” vs. “Hello word”).
string text_context;
// The maximum number of alternative predictions to generate.
uint32 alternatives;
};
// Represents the struct for query features supported by the backend handwriting
// recognizer.
// Corresponds to handwriting_feature_query.idl.
struct HandwritingFeatureQuery {
// If non-empty, queries whether the languages are supported.
// Languages are IETF BCP 47 language tags, e.g., "en", "zh-CN", "zh-Hans".
array<string> languages;
// Whether the "alternative" feature is queried.
bool alternatives;
// Whether the "segmentation" feature is queried.
bool segmentation_result;
};
// Because "bool" is not nullable, we need to define an enum to present the
// status. And this will also be helpful when we need to extend it, for
// example, adding `kNeedDownload`.
// Add `kNotQueried` to make it possible to use `mojo::Converter` to convert
// `HandwritingFeatureQueryResult` types (otherwise, there is no way to know
// whether a feature is queried or not in the converter function.
enum HandwritingFeatureStatus {
kNotQueried,
kSupported,
kNotSupported,
};
// The feature query result.
// Corresponds to handwriting_feature_query_result.idl.
struct HandwritingFeatureQueryResult {
HandwritingFeatureStatus languages = kNotQueried;
HandwritingFeatureStatus alternatives = kNotQueried;
HandwritingFeatureStatus segmentation_result = kNotQueried;
};
// Used in creating recognizer.
// Corresponds to handwriting_model_constraint.idl.
struct HandwritingModelConstraint {
// Languages are IETF BCP 47 language tags, e.g., "en", "zh-CN", "zh-Hans".
array<string> languages;
};
// Interface for a renderer to use a specific speech recognition backend.
// The browser handles the requests and forwards them to the appropriate
// backend.
interface HandwritingRecognizer {
// Does the recognition and outputs the prediction result.
// This is used by IDL API `blink::HandwritingDrawing::getPrediction`.
// The input `strokes` and `hints` should both come from
// `blink::HandwritingDrawing`.
// If the returned `Optional` has no value, it means there is some error in
// recognition. If the returned `Optional` has value but the array is empty,
// it means the recognizer can not recognize anything from the input.
GetPrediction(array<HandwritingStroke> strokes, HandwritingHints hints)
=> (array<HandwritingPrediction>? prediction);
};
// Per-document interface that allows the renderer to ask the browser for
// a specific speech recognizer backend, query capabilities, et cetera.
// Corresponds to `navigator_handwriting_recognition_service.idl`.
interface HandwritingRecognitionService {
// Creates a recognizer.
// This is used by IDL API
// `blink::HandwritingRecognitionService::createHandwritingRecognizer`.
CreateHandwritingRecognizer(HandwritingModelConstraint constraint)
=> (CreateHandwritingRecognizerResult result,
pending_remote<HandwritingRecognizer>? handwriting_recognizer);
// Queries the features supported by the platform.
// This is used by IDL API
// `blink::HandwritingRecognitionService::queryHandwritingRecognizerSupport`.
QueryHandwritingRecognizerSupport(HandwritingFeatureQuery query)
=> (HandwritingFeatureQueryResult? result);
};
enum CreateHandwritingRecognizerResult {
kOk,
kError,
};