protos

type ModelProto struct {

	// Sentence pieces with scores.
	Pieces []*ModelProto_SentencePiece `protobuf:"bytes,1,rep,name=pieces" json:"pieces,omitempty"`
	// Spec used to generate this model file.
	TrainerSpec *TrainerSpec `protobuf:"bytes,2,opt,name=trainer_spec,json=trainerSpec" json:"trainer_spec,omitempty"`
	// Spec for text normalization.
	NormalizerSpec *NormalizerSpec `protobuf:"bytes,3,opt,name=normalizer_spec,json=normalizerSpec" json:"normalizer_spec,omitempty"`
	// Stores sample input and its expected segmentation to verify the model.
	SelfTestData *SelfTestData `protobuf:"bytes,4,opt,name=self_test_data,json=selfTestData" json:"self_test_data,omitempty"`
	// Spec for text de-normalization.
	DenormalizerSpec *NormalizerSpec `protobuf:"bytes,5,opt,name=denormalizer_spec,json=denormalizerSpec" json:"denormalizer_spec,omitempty"`
	// contains filtered or unexported fields
}

type ModelProto_SentencePiece struct {
	Piece *string                        `protobuf:"bytes,1,opt,name=piece" json:"piece,omitempty"` // piece must not be empty.
	Score *float32                       `protobuf:"fixed32,2,opt,name=score" json:"score,omitempty"`
	Type  *ModelProto_SentencePiece_Type `` /* 142-byte string literal not displayed */
	// contains filtered or unexported fields
}

type ModelProto_SentencePiece_Type int32

type NormalizerSpec struct {

	// name of normalization rule.
	Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"`
	// Pre-compiled normalization rule created by
	// Builder::GetPrecompiledCharsMap() or Builder::CompileCharsMap() method.
	// Usually this field is set by Builder::GetNormalizerSpec() method.
	PrecompiledCharsmap []byte `protobuf:"bytes,2,opt,name=precompiled_charsmap,json=precompiledCharsmap" json:"precompiled_charsmap,omitempty"`
	// Adds dummy whitespace at the beginning of text in order to
	// treat "world" in "world" and "hello world" in the same way.
	AddDummyPrefix *bool `protobuf:"varint,3,opt,name=add_dummy_prefix,json=addDummyPrefix,def=1" json:"add_dummy_prefix,omitempty"`
	// Removes leading, trailing, and duplicate internal whitespace.
	RemoveExtraWhitespaces *bool `` /* 129-byte string literal not displayed */
	// Replaces whitespace with meta symbol.
	// This field must be true to train sentence piece model.
	EscapeWhitespaces *bool `protobuf:"varint,5,opt,name=escape_whitespaces,json=escapeWhitespaces,def=1" json:"escape_whitespaces,omitempty"`
	// Custom normalization rule file in TSV format.
	// https://github.com/google/sentencepiece/blob/master/doc/normalization.md
	// This field is only used in SentencePieceTrainer::Train() method, which
	// compiles the rule into the binary rule stored in `precompiled_charsmap`.
	NormalizationRuleTsv *string `protobuf:"bytes,6,opt,name=normalization_rule_tsv,json=normalizationRuleTsv" json:"normalization_rule_tsv,omitempty"`
	// contains filtered or unexported fields
}

type SelfTestData struct {
	Samples []*SelfTestData_Sample `protobuf:"bytes,1,rep,name=samples" json:"samples,omitempty"`
	// contains filtered or unexported fields
}

type SelfTestData_Sample struct {
	Input    *string `protobuf:"bytes,1,opt,name=input" json:"input,omitempty"`
	Expected *string `protobuf:"bytes,2,opt,name=expected" json:"expected,omitempty"`
	// contains filtered or unexported fields
}

type TrainerSpec struct {

	// /////////////////////////////////////////////////////////////////
	// General parameters
	//
	// Input corpus files.
	//
	//	Trainer accepts the following two formats:
	//	A) Monolingual: plain text, one sentence per line.
	//	B) Bilingual:   TSV, source sentence <tab> target sentence
	//	When bilingual data is passed, shared vocabulary model is built.
	//	Note that the input file must be raw corpus, not a preprocessed corpus.
	//	Trainer only loads the first `input_sentence_size` sentences specified
	//	with this parameter.
	Input []string `protobuf:"bytes,1,rep,name=input" json:"input,omitempty"`
	// Input corpus format:
	// "text": one-sentence-per-line text format (default)
	// "tsv":  sentence <tab> freq
	InputFormat *string `protobuf:"bytes,7,opt,name=input_format,json=inputFormat" json:"input_format,omitempty"`
	// Output model file prefix.
	// <model_prefix>.model and <model_prefix>.vocab are generated.
	ModelPrefix *string                `protobuf:"bytes,2,opt,name=model_prefix,json=modelPrefix" json:"model_prefix,omitempty"`
	ModelType   *TrainerSpec_ModelType `` /* 161-byte string literal not displayed */
	// Vocabulary size. 8k is the default size.
	VocabSize *int32 `protobuf:"varint,4,opt,name=vocab_size,json=vocabSize,def=8000" json:"vocab_size,omitempty"`
	// List of the languages this model can accept.
	// Since the model is language-agnostic, this field is used as a reference.
	AcceptLanguage []string `protobuf:"bytes,5,rep,name=accept_language,json=acceptLanguage" json:"accept_language,omitempty"`
	// Size of self-test samples, which are encoded in the model file.
	SelfTestSampleSize *int32 `protobuf:"varint,6,opt,name=self_test_sample_size,json=selfTestSampleSize,def=0" json:"self_test_sample_size,omitempty"`
	// Whether to use DP version of sentencepiece. Use it with TSV input format
	// (requires precomputed word tab counts to work).
	EnableDifferentialPrivacy *bool `` /* 139-byte string literal not displayed */
	// Set these parameters if you need DP version of sentencepiece.
	// std of noise to add.
	DifferentialPrivacyNoiseLevel *float32 `` /* 154-byte string literal not displayed */
	// Clipping threshold to apply after adding noise. All the words with
	// frequency less than this value are dropped.
	DifferentialPrivacyClippingThreshold *uint64 `` /* 174-byte string literal not displayed */
	// /////////////////////////////////////////////////////////////////
	// Training parameters.
	//
	// Uses characters which cover the corpus with the ratio of `chars_coverage`.
	// This parameter determines the set of basic Alphabet of sentence piece.
	// 1.0 - `chars_coverage` characters are treated as UNK.
	// See also required_chars field.
	CharacterCoverage *float32 `protobuf:"fixed32,10,opt,name=character_coverage,json=characterCoverage,def=0.9995" json:"character_coverage,omitempty"`
	// Maximum size of sentences the trainer loads from `input` parameter.
	// Trainer simply loads the `input` files in sequence.
	// It is better to shuffle the input corpus randomly.
	InputSentenceSize    *uint64 `protobuf:"varint,11,opt,name=input_sentence_size,json=inputSentenceSize,def=0" json:"input_sentence_size,omitempty"`
	ShuffleInputSentence *bool   `protobuf:"varint,19,opt,name=shuffle_input_sentence,json=shuffleInputSentence,def=1" json:"shuffle_input_sentence,omitempty"`
	// Maximum size of sentences to make seed sentence pieces.
	// Extended suffix array is constructed to extract frequent
	// sub-strings from the corpus. This uses 20N working space,
	// where N is the size of corpus.
	//
	// Deprecated: Marked as deprecated in com_github_gomlx_go-huggingface_tokenizers_sentencepiece_private_protos/sentencepiece_model.proto.
	MiningSentenceSize *int32 `protobuf:"varint,12,opt,name=mining_sentence_size,json=miningSentenceSize" json:"mining_sentence_size,omitempty"`
	// Maximum size of sentences to train sentence pieces.
	//
	// Deprecated: Marked as deprecated in com_github_gomlx_go-huggingface_tokenizers_sentencepiece_private_protos/sentencepiece_model.proto.
	TrainingSentenceSize *int32 `protobuf:"varint,13,opt,name=training_sentence_size,json=trainingSentenceSize" json:"training_sentence_size,omitempty"`
	// The size of seed sentencepieces.
	// `seed_sentencepiece_size` must be larger than `vocab_size`.
	SeedSentencepieceSize *int32 `` /* 133-byte string literal not displayed */
	// In every EM sub-iterations, keeps top
	// `shrinking_factor` * `current sentencepieces size` with respect to
	// the loss of the sentence piece. This value should be smaller than 1.0.
	ShrinkingFactor *float32 `protobuf:"fixed32,15,opt,name=shrinking_factor,json=shrinkingFactor,def=0.75" json:"shrinking_factor,omitempty"`
	// The maximum sentence length in byte. The sentences with the length
	// larger than `max_sentence_length` is simply ignored.
	// Longer input tends to bring the following risks:
	//   - Overflow during EM training (unigram language model only)
	//   - Performance drop because of O(n log n) cost in BPE.
	MaxSentenceLength *int32 `protobuf:"varint,18,opt,name=max_sentence_length,json=maxSentenceLength,def=4192" json:"max_sentence_length,omitempty"`
	// Number of threads in the training.
	NumThreads *int32 `protobuf:"varint,16,opt,name=num_threads,json=numThreads,def=16" json:"num_threads,omitempty"`
	// Number of EM sub iterations.
	NumSubIterations *int32 `protobuf:"varint,17,opt,name=num_sub_iterations,json=numSubIterations,def=2" json:"num_sub_iterations,omitempty"`
	// /////////////////////////////////////////////////////////////////
	// SentencePiece parameters which control the shapes of sentence piece.
	//
	// Maximum length of sentencepiece.
	MaxSentencepieceLength *int32 `` /* 131-byte string literal not displayed */
	// Uses Unicode script to split sentence pieces.
	// When `split_by_unicode_script` is true, we do not allow sentence piece to
	// include multiple Unicode scripts, e.g. "F1" is not a valid piece.
	// Exception: CJ characters (Hiragana/Katakana/Han) are all handled
	// as one script type, since Japanese word can consist of multiple scripts.
	// This exception is always applied regardless of the accept-language
	// parameter.
	SplitByUnicodeScript *bool `` /* 126-byte string literal not displayed */
	// When `split_by_number` is true, put a boundary between number and
	// non-number transition. If we want to treat "F1" is one token, set this flag
	// to be false.
	SplitByNumber *bool `protobuf:"varint,23,opt,name=split_by_number,json=splitByNumber,def=1" json:"split_by_number,omitempty"`
	// Use a white space to split sentence pieces.
	// When `split_by_whitespace` is false, we may have the piece containing
	// a white space in the middle. e.g., "in_the".
	SplitByWhitespace *bool `protobuf:"varint,22,opt,name=split_by_whitespace,json=splitByWhitespace,def=1" json:"split_by_whitespace,omitempty"`
	// Adds whitespace symbol (_) as a suffix instead of prefix. e.g., _hello =>
	// hello_. When `treat_whitespace_as_suffix` is true,
	// NormalizerSpec::add_dummy_prefix will add the dummy whitespace to the end
	// of sentence.
	TreatWhitespaceAsSuffix *bool `` /* 135-byte string literal not displayed */
	// Allows pieces that only contain whitespaces instead of appearing only as
	// prefix or suffix of other pieces.
	AllowWhitespaceOnlyPieces *bool `` /* 141-byte string literal not displayed */
	// Split all digits (0-9) into separate pieces.
	SplitDigits *bool `protobuf:"varint,25,opt,name=split_digits,json=splitDigits,def=0" json:"split_digits,omitempty"`
	// Defines the pre-tokenization delimiter.
	// When specified, no pieces crossing this delimiter is not included
	// in the vocab. Then the delimiter string is virtually ignored
	// during the training. This field can allows constraints on the vocabulary
	// selection. Note that this field is available on unigram mode.
	PretokenizationDelimiter *string `` /* 132-byte string literal not displayed */
	// /////////////////////////////////////////////////////////////////
	// Vocabulary management
	//
	// Defines control symbols used as an indicator to
	// change the behavior of the decoder. <s> and </s> are pre-defined.
	// We can use this field to encode various meta information,
	// including language indicator in multilingual model.
	// These symbols are not visible to users, but visible to
	// the decoder. Note that when the input sentence contains control symbols,
	// they are not treated as one token, but segmented into normal pieces.
	// Control symbols must be inserted independently from the segmentation.
	ControlSymbols []string `protobuf:"bytes,30,rep,name=control_symbols,json=controlSymbols" json:"control_symbols,omitempty"`
	// Defines user defined symbols.
	// These symbols are added with extremely high score
	// so they are always treated as one unique symbol in any context.
	// Typical usage of user_defined_symbols is placeholder for named entities.
	UserDefinedSymbols []string `protobuf:"bytes,31,rep,name=user_defined_symbols,json=userDefinedSymbols" json:"user_defined_symbols,omitempty"`
	// Defines required characters. Each UTF8 character in this string is included
	// in the character set regardless of character_coverage value. Unlike
	// user_defined_symbols, these characters have scores based on the frequency
	// on input sentences, and the model can form subwords using characters
	// in this field.
	RequiredChars *string `protobuf:"bytes,36,opt,name=required_chars,json=requiredChars" json:"required_chars,omitempty"`
	// Decomposes unknown pieces into UTF-8 bytes.
	ByteFallback *bool `protobuf:"varint,35,opt,name=byte_fallback,json=byteFallback,def=0" json:"byte_fallback,omitempty"`
	// When creating the vocabulary file, defines whether or not to additionally
	// output the score for each piece.
	VocabularyOutputPieceScore *bool `` /* 144-byte string literal not displayed */
	// `vocab_size` is treated as hard limit. Crash if
	// the model can not produce the vocab of size `vocab_size`,
	// When `hard_vocab_limit` is false, vocab_size is treated
	// as soft limit. Note that when model_type=char,
	// always assumes hard_vocab_limit = false.
	HardVocabLimit *bool `protobuf:"varint,33,opt,name=hard_vocab_limit,json=hardVocabLimit,def=1" json:"hard_vocab_limit,omitempty"`
	// use all symbols for vocab extraction. This flag is valid
	// if model type is either CHAR or WORD
	UseAllVocab *bool `protobuf:"varint,34,opt,name=use_all_vocab,json=useAllVocab,def=0" json:"use_all_vocab,omitempty"`
	// /////////////////////////////////////////////////////////////////
	// Reserved special meta tokens.
	// * -1 is not used.
	// * unk_id must not be -1.
	// Id must starts with 0 and be contigous.
	UnkId    *int32  `protobuf:"varint,40,opt,name=unk_id,json=unkId,def=0" json:"unk_id,omitempty"`  // <unk>
	BosId    *int32  `protobuf:"varint,41,opt,name=bos_id,json=bosId,def=1" json:"bos_id,omitempty"`  // <s>
	EosId    *int32  `protobuf:"varint,42,opt,name=eos_id,json=eosId,def=2" json:"eos_id,omitempty"`  // </s>
	PadId    *int32  `protobuf:"varint,43,opt,name=pad_id,json=padId,def=-1" json:"pad_id,omitempty"` // <pad> (padding)
	UnkPiece *string `protobuf:"bytes,45,opt,name=unk_piece,json=unkPiece,def=<unk>" json:"unk_piece,omitempty"`
	BosPiece *string `protobuf:"bytes,46,opt,name=bos_piece,json=bosPiece,def=<s>" json:"bos_piece,omitempty"`
	EosPiece *string `protobuf:"bytes,47,opt,name=eos_piece,json=eosPiece,def=</s>" json:"eos_piece,omitempty"`
	PadPiece *string `protobuf:"bytes,48,opt,name=pad_piece,json=padPiece,def=<pad>" json:"pad_piece,omitempty"`
	// Encodes <unk> into U+2047 (DOUBLE QUESTION MARK),
	// since this character can be useful both for user and
	// developer. We can easily figure out that <unk> is emitted.
	UnkSurface *string `protobuf:"bytes,44,opt,name=unk_surface,json=unkSurface,def= ⁇ " json:"unk_surface,omitempty"`
	// Increase bit depth to allow unigram model training on large
	// (>10M sentences) corpora. A Side-effect of enabling this flag
	// is increased memory usage.
	TrainExtremelyLargeCorpus *bool `` /* 141-byte string literal not displayed */
	// Path to a seed sentencepieces file, with one tab-separated
	// seed sentencepiece <tab> frequency per line.
	SeedSentencepiecesFile *string `` /* 128-byte string literal not displayed */
	// contains filtered or unexported fields
}

type TrainerSpec_ModelType int32