diff --git a/Cargo.toml b/Cargo.toml index 7f7a6aa8..e35daef2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,7 +25,7 @@ ttrpc-codegen = { version = "0.6.0", path = "./ttrpc-codegen" } ttrpc-compiler = { version = "0.8.0", path = "./compiler" } protobuf = "3.7.2" protobuf-codegen = "3.7.2" -protobuf-support = "3.7.2" +protobuf-parse = "3.7.2" [package] name = "ttrpc" diff --git a/README.md b/README.md index 692e2196..cd354500 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,9 @@ fn main() { } ``` +Canonical Google well-known type imports, such as `google/protobuf/timestamp.proto`, are available +automatically when generating code programmatically and do not require an additional include path. + # async/.await ttrpc-rust supports async/.await. By using async/.await you can reduce the overhead and resource consumption caused by threads. diff --git a/compiler/README.md b/compiler/README.md index 1882f19b..56c827f2 100644 --- a/compiler/README.md +++ b/compiler/README.md @@ -7,6 +7,12 @@ generate rust version ttrpc code from proto files. - [Manual Generation](https://github.com/containerd/ttrpc-rust#1-generate-with-protoc-command) uses ttrpc-compiler as a protoc plugin - [Programmatic Generation](https://github.com/containerd/ttrpc-rust#2-generate-programmatically) uses ttrpc-compiler as a rust crate +## Well-known types + +RPC inputs and outputs from canonical Google well-known proto dependencies reference the +corresponding types provided by the `protobuf` runtime. Well-known proto files explicitly selected +for generation continue to use their locally generated modules. + ## Versions | ttrpc-compiler version | ttrpc version | | ------------- | ------------- | diff --git a/compiler/src/codegen.rs b/compiler/src/codegen.rs index 4f6aa464..41fb3621 100644 --- a/compiler/src/codegen.rs +++ b/compiler/src/codegen.rs @@ -43,7 +43,10 @@ use std::{ }; use crate::{ - util::proto_path_to_rust_mod, util::scope::RootScope, util::writer::CodeWriter, Customize, + util::proto_path_to_rust_mod, + util::scope::{RootScope, RustType}, + util::writer::CodeWriter, + Customize, }; use protobuf::{ descriptor::*, @@ -88,21 +91,28 @@ impl<'a> MethodGen<'a> { } fn input(&self) -> String { - format!( - "super::{}", - self.root_scope - .find_message(self.proto.input_type()) - .rust_fq_name() - ) + self.root_scope + .rust_type(self.proto.input_type()) + .to_string() } fn output(&self) -> String { - format!( - "super::{}", - self.root_scope - .find_message(self.proto.output_type()) - .rust_fq_name() - ) + self.root_scope + .rust_type(self.proto.output_type()) + .to_string() + } + + fn request_handler_call(&self, macro_name: &str, request: &str) -> String { + match self.root_scope.rust_type(self.proto.input_type()) { + RustType::Generated { module, name } => format!( + "::ttrpc::{macro_name}!(self, ctx, {request}, {module}, {name}, {});", + self.name() + ), + rust_type @ RustType::ProtobufRuntime { .. } => format!( + "::ttrpc::{macro_name}!(self, ctx, {request}, {rust_type}, {});", + self.name() + ), + } } fn method_type(&self) -> (MethodType, String) { @@ -164,10 +174,7 @@ impl<'a> MethodGen<'a> { |w| { w.block("fn handler(&self, ctx: ::ttrpc::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<()> {", "}", |w| { - w.write_line(format!("::ttrpc::request_handler!(self, ctx, req, {}, {}, {});", - proto_path_to_rust_mod(self.root_scope.find_message(self.proto.input_type()).fd.name()), - self.root_scope.find_message(self.proto.input_type()).rust_name(), - self.name())); + w.write_line(self.request_handler_call("request_handler", "req")); w.write_line("Ok(())"); }); }); @@ -181,10 +188,7 @@ impl<'a> MethodGen<'a> { |w| { w.block("async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, req: ::ttrpc::Request) -> ::ttrpc::Result<::ttrpc::Response> {", "}", |w| { - w.write_line(format!("::ttrpc::async_request_handler!(self, ctx, req, {}, {}, {});", - proto_path_to_rust_mod(self.root_scope.find_message(self.proto.input_type()).fd.name()), - self.root_scope.find_message(self.proto.input_type()).rust_name(), - self.name())); + w.write_line(self.request_handler_call("async_request_handler", "req")); }); }); } @@ -205,10 +209,7 @@ impl<'a> MethodGen<'a> { |w| { w.block("async fn handler(&self, ctx: ::ttrpc::r#async::TtrpcContext, mut inner: ::ttrpc::r#async::StreamInner) -> ::ttrpc::Result> {", "}", |w| { - w.write_line(format!("::ttrpc::async_server_streamimg_handler!(self, ctx, inner, {}, {}, {});", - proto_path_to_rust_mod(self.root_scope.find_message(self.proto.input_type()).fd.name()), - self.root_scope.find_message(self.proto.input_type()).rust_name(), - self.name())); + w.write_line(self.request_handler_call("async_server_streamimg_handler", "inner")); }); }); } @@ -712,7 +713,7 @@ pub fn gen( let files_map: HashMap<&str, &FileDescriptorProto> = file_descriptors.iter().map(|f| (f.name(), f)).collect(); - let root_scope = RootScope { file_descriptors }; + let root_scope = RootScope::new(file_descriptors, files_to_generate); let mut results = CodeGeneratorResponse::new(); results.set_supported_features(CodeGeneratorResponse_Feature::FEATURE_PROTO3_OPTIONAL as _); diff --git a/compiler/src/util/scope.rs b/compiler/src/util/scope.rs index 53c40f3c..58c3a81e 100644 --- a/compiler/src/util/scope.rs +++ b/compiler/src/util/scope.rs @@ -2,9 +2,28 @@ //! the protobuf / protobuf-codegen crates, but were then removed. //! The missing functionalities have been reimplemented in this module. +use std::fmt; + use protobuf::descriptor::{DescriptorProto, FileDescriptorProto}; -// vendered from https://github.com/stepancheg/rust-protobuf/blob/v3.7.2/protobuf-codegen/src/gen/rust/keywords.rs +use super::to_snake_case; + +const DESCRIPTOR_PROTO_FILE: &str = "google/protobuf/descriptor.proto"; + +const WELL_KNOWN_TYPE_PROTO_FILES: &[&str] = &[ + "google/protobuf/any.proto", + "google/protobuf/api.proto", + "google/protobuf/duration.proto", + "google/protobuf/empty.proto", + "google/protobuf/field_mask.proto", + "google/protobuf/source_context.proto", + "google/protobuf/struct.proto", + "google/protobuf/timestamp.proto", + "google/protobuf/type.proto", + "google/protobuf/wrappers.proto", +]; + +// vendored from https://github.com/stepancheg/rust-protobuf/blob/v3.7.2/protobuf-codegen/src/gen/rust/keywords.rs fn is_rust_keyword(ident: &str) -> bool { #[rustfmt::skip] static RUST_KEYWORDS: &[&str] = &[ @@ -71,7 +90,8 @@ fn is_rust_keyword(ident: &str) -> bool { // reimplementation based on https://github.com/stepancheg/rust-protobuf/blob/v3.7.2/protobuf-codegen/src/gen/scope.rs#L26 // it only implements the `find_message` method with not extra dependencies pub struct RootScope<'a> { - pub file_descriptors: &'a [FileDescriptorProto], + file_descriptors: &'a [FileDescriptorProto], + files_to_generate: &'a [String], } // re-implementation of https://github.com/stepancheg/rust-protobuf/blob/v3.7.2/protobuf-codegen/src/gen/scope.rs#L340 @@ -82,6 +102,21 @@ pub struct ScopedMessage<'a> { pub msg: &'a DescriptorProto, } +#[derive(Debug, Eq, PartialEq)] +pub enum RustType { + Generated { module: String, name: String }, + ProtobufRuntime { path: String }, +} + +impl fmt::Display for RustType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Generated { module, name } => write!(f, "super::{module}::{name}"), + Self::ProtobufRuntime { path } => f.write_str(path), + } + } +} + impl ScopedMessage<'_> { pub fn prefix(&self) -> String { let mut prefix = String::new(); @@ -108,17 +143,57 @@ impl ScopedMessage<'_> { r } - // fully-qualified name of this type - pub fn rust_fq_name(&self) -> String { - format!( - "{}::{}", - super::proto_path_to_rust_mod(self.fd.name()), - self.rust_name() - ) + fn protobuf_runtime_path(&self) -> String { + let mut path = String::new(); + for message in &self.path { + path.push_str(&to_snake_case(message.name())); + path.push_str("::"); + } + path.push_str(self.msg.name()); + path } } impl<'a> RootScope<'a> { + pub fn new( + file_descriptors: &'a [FileDescriptorProto], + files_to_generate: &'a [String], + ) -> Self { + Self { + file_descriptors, + files_to_generate, + } + } + + pub fn rust_type(&'a self, fqn: impl AsRef) -> RustType { + let message = self.find_message(fqn); + let file_name = message.fd.name(); + let module = super::proto_path_to_rust_mod(file_name); + let name = message.rust_name(); + + if !self.files_to_generate.iter().any(|file| file == file_name) { + if file_name == DESCRIPTOR_PROTO_FILE { + return RustType::ProtobufRuntime { + path: format!( + "::protobuf::descriptor::{}", + message.protobuf_runtime_path() + ), + }; + } + + if WELL_KNOWN_TYPE_PROTO_FILES.contains(&file_name) { + return RustType::ProtobufRuntime { + path: format!( + "::protobuf::well_known_types::{module}::{}", + message.protobuf_runtime_path() + ), + }; + } + } + + RustType::Generated { module, name } + } + pub fn find_message(&'a self, fqn: impl AsRef) -> ScopedMessage<'a> { let Some(fqn1) = fqn.as_ref().strip_prefix(".") else { panic!("name must start with dot: {}", fqn.as_ref()) @@ -147,6 +222,124 @@ impl<'a> RootScope<'a> { } } } - panic!("enum not found by name: {}", fqn.as_ref()) + panic!("message not found by name: {}", fqn.as_ref()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn file_descriptor(name: &str, package: &str, message: &str) -> FileDescriptorProto { + let mut descriptor = FileDescriptorProto::new(); + descriptor.set_name(name.to_owned()); + descriptor.set_package(package.to_owned()); + + let mut message_descriptor = DescriptorProto::new(); + message_descriptor.set_name(message.to_owned()); + descriptor.message_type.push(message_descriptor); + descriptor + } + + #[test] + fn well_known_dependencies_use_protobuf_runtime() { + let cases = [ + ("any.proto", "Any", "any"), + ("api.proto", "Api", "api"), + ("duration.proto", "Duration", "duration"), + ("empty.proto", "Empty", "empty"), + ("field_mask.proto", "FieldMask", "field_mask"), + ("source_context.proto", "SourceContext", "source_context"), + ("struct.proto", "Struct", "struct_"), + ("timestamp.proto", "Timestamp", "timestamp"), + ("type.proto", "Type", "type_"), + ("wrappers.proto", "StringValue", "wrappers"), + ]; + let files_to_generate = ["service.proto".to_owned()]; + + for (proto, message, module) in cases { + let descriptors = [file_descriptor( + &format!("google/protobuf/{proto}"), + "google.protobuf", + message, + )]; + let scope = RootScope::new(&descriptors, &files_to_generate); + + assert_eq!( + format!("::protobuf::well_known_types::{module}::{message}"), + scope + .rust_type(format!(".google.protobuf.{message}")) + .to_string() + ); + } + } + + #[test] + fn explicitly_generated_well_known_type_uses_local_module() { + let descriptors = [file_descriptor( + "google/protobuf/timestamp.proto", + "google.protobuf", + "Timestamp", + )]; + let files_to_generate = ["google/protobuf/timestamp.proto".to_owned()]; + let scope = RootScope::new(&descriptors, &files_to_generate); + + assert_eq!( + "super::timestamp::Timestamp", + scope.rust_type(".google.protobuf.Timestamp").to_string() + ); + } + + #[test] + fn descriptor_dependency_uses_protobuf_runtime() { + let descriptors = [file_descriptor( + DESCRIPTOR_PROTO_FILE, + "google.protobuf", + "FileDescriptorProto", + )]; + let files_to_generate = ["service.proto".to_owned()]; + let scope = RootScope::new(&descriptors, &files_to_generate); + + assert_eq!( + "::protobuf::descriptor::FileDescriptorProto", + scope + .rust_type(".google.protobuf.FileDescriptorProto") + .to_string() + ); + } + + #[test] + fn nested_descriptor_dependency_uses_protobuf_runtime() { + let mut descriptor = + file_descriptor(DESCRIPTOR_PROTO_FILE, "google.protobuf", "DescriptorProto"); + let mut nested = DescriptorProto::new(); + nested.set_name("ExtensionRange".to_owned()); + descriptor.message_type[0].nested_type.push(nested); + let descriptors = [descriptor]; + let files_to_generate = ["service.proto".to_owned()]; + let scope = RootScope::new(&descriptors, &files_to_generate); + + assert_eq!( + "::protobuf::descriptor::descriptor_proto::ExtensionRange", + scope + .rust_type(".google.protobuf.DescriptorProto.ExtensionRange") + .to_string() + ); + } + + #[test] + fn non_well_known_google_type_uses_local_module() { + let descriptors = [file_descriptor( + "google/protobuf/custom.proto", + "google.protobuf", + "Custom", + )]; + let files_to_generate = ["service.proto".to_owned()]; + let scope = RootScope::new(&descriptors, &files_to_generate); + + assert_eq!( + "super::custom::Custom", + scope.rust_type(".google.protobuf.Custom").to_string() + ); } } diff --git a/example/build.rs b/example/build.rs index 52851c6f..ed837668 100644 --- a/example/build.rs +++ b/example/build.rs @@ -20,6 +20,7 @@ fn main() { "protocols/protos/health.proto", "protocols/protos/google/protobuf/empty.proto", "protocols/protos/oci.proto", + "protocols/protos/well_known.proto", ]; let protobuf_customized = ProtobufCustomize::default().gen_mod_rs(true); @@ -38,6 +39,7 @@ fn main() { // Only async support stream currently. protos.push("protocols/protos/streaming.proto"); + protos.push("protocols/protos/well_known_streaming.proto"); Codegen::new() .out_dir("protocols/asynchronous") diff --git a/example/protocols/protos/well_known.proto b/example/protocols/protos/well_known.proto new file mode 100644 index 00000000..8f3a5751 --- /dev/null +++ b/example/protocols/protos/well_known.proto @@ -0,0 +1,16 @@ +syntax = "proto3"; + +package well_known; + +import "google/protobuf/descriptor.proto"; +import "google/protobuf/timestamp.proto"; + +message TimestampedRequest { + google.protobuf.Timestamp created_at = 1; +} + +service Clock { + rpc Now(google.protobuf.Timestamp) returns (google.protobuf.Timestamp); + rpc Describe(google.protobuf.FileDescriptorProto) returns (google.protobuf.FileDescriptorProto); + rpc Echo(TimestampedRequest) returns (TimestampedRequest); +} diff --git a/example/protocols/protos/well_known_streaming.proto b/example/protocols/protos/well_known_streaming.proto new file mode 100644 index 00000000..32ecd3e5 --- /dev/null +++ b/example/protocols/protos/well_known_streaming.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; + +package well_known_streaming; + +import "google/protobuf/timestamp.proto"; + +service Clock { + rpc Watch(google.protobuf.Timestamp) returns (stream google.protobuf.Timestamp); + rpc Collect(stream google.protobuf.Timestamp) returns (google.protobuf.Timestamp); + rpc Exchange(stream google.protobuf.Timestamp) returns (stream google.protobuf.Timestamp); +} diff --git a/src/asynchronous/utils.rs b/src/asynchronous/utils.rs index ce315edb..d3b8c601 100644 --- a/src/asynchronous/utils.rs +++ b/src/asynchronous/utils.rs @@ -14,8 +14,8 @@ use crate::proto::{MessageHeader, Request, Response}; /// Handle request in async mode. #[macro_export] macro_rules! async_request_handler { - ($class: ident, $ctx: ident, $req: ident, $server: ident, $req_type: ident, $req_fn: ident) => { - let mut req = super::$server::$req_type::new(); + ($class: ident, $ctx: ident, $req: ident, $req_type: path, $req_fn: ident) => { + let mut req = <$req_type>::new(); { let mut s = CodedInputStream::from_bytes(&$req.payload); req.merge_from(&mut s) @@ -47,6 +47,15 @@ macro_rules! async_request_handler { return Ok(res); }; + ($class: ident, $ctx: ident, $req: ident, $server: ident, $req_type: ident, $req_fn: ident) => { + $crate::async_request_handler!( + $class, + $ctx, + $req, + super::$server::$req_type, + $req_fn + ) + }; } /// Handle client streaming in async mode. @@ -83,9 +92,9 @@ macro_rules! async_client_streamimg_handler { /// Handle server streaming in async mode. #[macro_export] macro_rules! async_server_streamimg_handler { - ($class: ident, $ctx: ident, $inner: ident, $server: ident, $req_type: ident, $req_fn: ident) => { + ($class: ident, $ctx: ident, $inner: ident, $req_type: path, $req_fn: ident) => { let req_buf = $inner.recv().await?; - let req = ::decode(&req_buf) + let req = <$req_type as ::ttrpc::proto::Codec>::decode(&req_buf) .map_err(|e| ::ttrpc::Error::Others(e.to_string()))?; let stream = ::ttrpc::r#async::ServerStreamSender::new($inner); match $class.service.$req_fn(&$ctx, req, stream).await { @@ -109,6 +118,15 @@ macro_rules! async_server_streamimg_handler { } } }; + ($class: ident, $ctx: ident, $inner: ident, $server: ident, $req_type: ident, $req_fn: ident) => { + $crate::async_server_streamimg_handler!( + $class, + $ctx, + $inner, + super::$server::$req_type, + $req_fn + ) + }; } /// Handle duplex streaming in async mode. diff --git a/src/sync/utils.rs b/src/sync/utils.rs index 616b615e..f4228f51 100644 --- a/src/sync/utils.rs +++ b/src/sync/utils.rs @@ -46,9 +46,9 @@ pub fn response_error_to_channel( /// Handle request in sync mode. #[macro_export] macro_rules! request_handler { - ($class: ident, $ctx: ident, $req: ident, $server: ident, $req_type: ident, $req_fn: ident) => { + ($class: ident, $ctx: ident, $req: ident, $req_type: path, $req_fn: ident) => { let mut s = CodedInputStream::from_bytes(&$req.payload); - let mut req = super::$server::$req_type::new(); + let mut req = <$req_type>::new(); req.merge_from(&mut s) .map_err(::ttrpc::err_to_others!(e, ""))?; @@ -76,6 +76,15 @@ macro_rules! request_handler { } ::ttrpc::response_to_channel($ctx.mh.stream_id, res, $ctx.res_tx)? }; + ($class: ident, $ctx: ident, $req: ident, $server: ident, $req_type: ident, $req_fn: ident) => { + $crate::request_handler!( + $class, + $ctx, + $req, + super::$server::$req_type, + $req_fn + ) + }; } /// Send request through sync client. diff --git a/ttrpc-codegen/Cargo.toml b/ttrpc-codegen/Cargo.toml index 6c92ba2e..996f78da 100644 --- a/ttrpc-codegen/Cargo.toml +++ b/ttrpc-codegen/Cargo.toml @@ -13,7 +13,10 @@ readme = "README.md" [dependencies] -protobuf-support = { workspace = true } protobuf = { workspace = true } protobuf-codegen = { workspace = true } +protobuf-parse = { workspace = true } ttrpc-compiler = { workspace = true } + +[dev-dependencies] +tempfile = "3" diff --git a/ttrpc-codegen/README.md b/ttrpc-codegen/README.md index 5194ac25..43049204 100644 --- a/ttrpc-codegen/README.md +++ b/ttrpc-codegen/README.md @@ -24,15 +24,25 @@ fn main() { .customize(Customize { ..Default::default() }) - .rust_protobuf_customize(ProtobufCustomize { - ..Default::default() - } + .rust_protobuf_customize(ProtobufCustomize::default()) .run() .expect("Gen code failed."); } ``` +## Well-known types + +Canonical Google well-known type imports such as `google/protobuf/timestamp.proto` and +`google/protobuf/empty.proto` are resolved automatically. They do not need to be copied into the +source tree or added through an extra include directory. + +When an imported well-known type is used as an RPC input or output, generated services reference +the type provided by the `protobuf` runtime. A well-known proto explicitly listed as an input keeps +using its locally generated module for compatibility. Proto definitions outside the standard +well-known type set, including Google API definitions, must still be available through an include +directory. + Cargo.toml: ``` diff --git a/ttrpc-codegen/src/convert.rs b/ttrpc-codegen/src/convert.rs deleted file mode 100644 index c01f1a42..00000000 --- a/ttrpc-codegen/src/convert.rs +++ /dev/null @@ -1,1058 +0,0 @@ -//! Convert parser model to rust-protobuf model - -use std::iter; - -use crate::model; - -use crate::str_lit::StrLitDecodeError; -use protobuf::Message; - -#[derive(Debug)] -pub enum ConvertError { - UnsupportedOption(String), - ExtensionNotFound(String), - WrongExtensionType(String, &'static str), - UnsupportedExtensionType(String, String), - StrLitDecodeError(StrLitDecodeError), - DefaultValueIsNotStringLiteral, - WrongOptionType, -} - -impl From for ConvertError { - fn from(e: StrLitDecodeError) -> Self { - ConvertError::StrLitDecodeError(e) - } -} - -pub type ConvertResult = Result; - -trait ProtobufOptions { - fn by_name(&self, name: &str) -> Option<&model::ProtobufConstant>; - - fn by_name_bool(&self, name: &str) -> ConvertResult> { - match self.by_name(name) { - Some(&model::ProtobufConstant::Bool(b)) => Ok(Some(b)), - Some(_) => Err(ConvertError::WrongOptionType), - None => Ok(None), - } - } -} - -impl ProtobufOptions for &[model::ProtobufOption] { - fn by_name(&self, name: &str) -> Option<&model::ProtobufConstant> { - let option_name = name; - for model::ProtobufOption { name, value } in *self { - if name == option_name { - return Some(value); - } - } - None - } -} - -enum MessageOrEnum { - Message, - Enum, -} - -impl MessageOrEnum { - fn descriptor_type(&self) -> protobuf::descriptor::field_descriptor_proto::Type { - match *self { - MessageOrEnum::Message => { - protobuf::descriptor::field_descriptor_proto::Type::TYPE_MESSAGE - } - MessageOrEnum::Enum => protobuf::descriptor::field_descriptor_proto::Type::TYPE_ENUM, - } - } -} - -#[derive(Debug, Eq, PartialEq, Clone)] -struct RelativePath { - path: String, -} - -impl RelativePath { - fn empty() -> RelativePath { - RelativePath::new(String::new()) - } - - fn new(path: String) -> RelativePath { - assert!(!path.starts_with('.')); - - RelativePath { path } - } - - fn is_empty(&self) -> bool { - self.path.is_empty() - } - - fn _last_part(&self) -> Option<&str> { - match self.path.rfind('.') { - Some(pos) => Some(&self.path[pos + 1..]), - None => { - if self.path.is_empty() { - None - } else { - Some(&self.path) - } - } - } - } - - fn parent(&self) -> Option { - match self.path.rfind('.') { - Some(pos) => Some(RelativePath::new(self.path[..pos].to_owned())), - None => { - if self.path.is_empty() { - None - } else { - Some(RelativePath::empty()) - } - } - } - } - - fn self_and_parents(&self) -> Vec { - let mut tmp = self.clone(); - - let mut r = vec![self.clone()]; - - while let Some(parent) = tmp.parent() { - r.push(parent.clone()); - tmp = parent; - } - - r - } - - fn append(&self, simple: &str) -> RelativePath { - if self.path.is_empty() { - RelativePath::new(simple.to_owned()) - } else { - RelativePath::new(format!("{}.{}", self.path, simple)) - } - } - - fn split_first_rem(&self) -> Option<(&str, RelativePath)> { - if self.is_empty() { - None - } else { - Some(match self.path.find('.') { - Some(dot) => ( - &self.path[..dot], - RelativePath::new(self.path[dot + 1..].to_owned()), - ), - None => (&self.path, RelativePath::empty()), - }) - } - } -} - -#[cfg(test)] -mod relative_path_test { - use super::*; - - #[test] - fn parent() { - assert_eq!(None, RelativePath::empty().parent()); - assert_eq!( - Some(RelativePath::empty()), - RelativePath::new("aaa".to_owned()).parent() - ); - assert_eq!( - Some(RelativePath::new("abc".to_owned())), - RelativePath::new("abc.def".to_owned()).parent() - ); - assert_eq!( - Some(RelativePath::new("abc.def".to_owned())), - RelativePath::new("abc.def.gh".to_owned()).parent() - ); - } - - #[test] - fn last_part() { - assert_eq!(None, RelativePath::empty()._last_part()); - assert_eq!( - Some("aaa"), - RelativePath::new("aaa".to_owned())._last_part() - ); - assert_eq!( - Some("def"), - RelativePath::new("abc.def".to_owned())._last_part() - ); - assert_eq!( - Some("gh"), - RelativePath::new("abc.def.gh".to_owned())._last_part() - ); - } - - #[test] - fn self_and_parents() { - assert_eq!( - vec![ - RelativePath::new("ab.cde.fghi".to_owned()), - RelativePath::new("ab.cde".to_owned()), - RelativePath::new("ab".to_owned()), - RelativePath::empty(), - ], - RelativePath::new("ab.cde.fghi".to_owned()).self_and_parents() - ); - } -} - -#[derive(Clone, Eq, PartialEq, Debug)] -struct AbsolutePath { - path: String, -} - -impl AbsolutePath { - fn root() -> AbsolutePath { - AbsolutePath::new(String::new()) - } - - fn new(path: String) -> AbsolutePath { - assert!(path.is_empty() || path.starts_with('.')); - assert!(!path.ends_with('.')); - AbsolutePath { path } - } - - fn from_path_without_dot(path: &str) -> AbsolutePath { - if path.is_empty() { - AbsolutePath::root() - } else { - assert!(!path.starts_with('.')); - assert!(!path.ends_with('.')); - AbsolutePath::new(format!(".{path}")) - } - } - - fn from_path_maybe_dot(path: &str) -> AbsolutePath { - if path.starts_with('.') { - AbsolutePath::new(path.to_owned()) - } else { - AbsolutePath::from_path_without_dot(path) - } - } - - fn push_simple(&mut self, simple: &str) { - assert!(!simple.is_empty()); - assert!(!simple.contains('.')); - self.path.push('.'); - self.path.push_str(simple); - } - - fn push_relative(&mut self, relative: &RelativePath) { - if !relative.is_empty() { - self.path.push('.'); - self.path.push_str(&relative.path); - } - } - - fn remove_prefix(&self, prefix: &AbsolutePath) -> Option { - if self.path.starts_with(&prefix.path) { - let rem = &self.path[prefix.path.len()..]; - if rem.is_empty() { - return Some(RelativePath::empty()); - } - if let Some(stripped) = rem.strip_prefix('.') { - return Some(RelativePath::new(stripped.to_string())); - } - } - None - } -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn absolute_path_push_simple() { - let mut foo = AbsolutePath::new(".foo".to_owned()); - foo.push_simple("bar"); - assert_eq!(AbsolutePath::new(".foo.bar".to_owned()), foo); - - let mut foo = AbsolutePath::root(); - foo.push_simple("bar"); - assert_eq!(AbsolutePath::new(".bar".to_owned()), foo); - } - - #[test] - fn absolute_path_remove_prefix() { - assert_eq!( - Some(RelativePath::empty()), - AbsolutePath::new(".foo".to_owned()) - .remove_prefix(&AbsolutePath::new(".foo".to_owned())) - ); - assert_eq!( - Some(RelativePath::new("bar".to_owned())), - AbsolutePath::new(".foo.bar".to_owned()) - .remove_prefix(&AbsolutePath::new(".foo".to_owned())) - ); - assert_eq!( - Some(RelativePath::new("baz.qux".to_owned())), - AbsolutePath::new(".foo.bar.baz.qux".to_owned()) - .remove_prefix(&AbsolutePath::new(".foo.bar".to_owned())) - ); - assert_eq!( - None, - AbsolutePath::new(".foo.barbaz".to_owned()) - .remove_prefix(&AbsolutePath::new(".foo.bar".to_owned())) - ); - } -} - -enum LookupScope<'a> { - File(&'a model::FileDescriptor), - Message(&'a model::Message), -} - -impl<'a> LookupScope<'a> { - fn messages(&self) -> &[model::Message] { - match *self { - LookupScope::File(file) => &file.messages, - LookupScope::Message(messasge) => &messasge.messages, - } - } - - fn find_message(&self, simple_name: &str) -> Option<&model::Message> { - self.messages().iter().find(|m| m.name == simple_name) - } - - fn enums(&self) -> &[model::Enumeration] { - match *self { - LookupScope::File(file) => &file.enums, - LookupScope::Message(messasge) => &messasge.enums, - } - } - - fn members(&self) -> Vec<(&str, MessageOrEnum)> { - let mut r = Vec::new(); - r.extend( - self.enums() - .iter() - .map(|e| (&e.name[..], MessageOrEnum::Enum)), - ); - r.extend( - self.messages() - .iter() - .map(|e| (&e.name[..], MessageOrEnum::Message)), - ); - r - } - - fn find_member(&self, simple_name: &str) -> Option { - self.members() - .into_iter() - .filter_map(|(member_name, message_or_enum)| { - if member_name == simple_name { - Some(message_or_enum) - } else { - None - } - }) - .next() - } - - fn resolve_message_or_enum( - &self, - current_path: &AbsolutePath, - path: &RelativePath, - ) -> Option<(AbsolutePath, MessageOrEnum)> { - let (first, rem) = path.split_first_rem()?; - - if rem.is_empty() { - match self.find_member(first) { - Some(message_or_enum) => { - let mut result_path = current_path.clone(); - result_path.push_simple(first); - Some((result_path, message_or_enum)) - } - None => None, - } - } else { - match self.find_message(first) { - Some(message) => { - let mut message_path = current_path.clone(); - message_path.push_simple(&message.name); - let message_scope = LookupScope::Message(message); - message_scope.resolve_message_or_enum(&message_path, &rem) - } - None => None, - } - } - } -} - -struct Resolver<'a> { - current_file: &'a model::FileDescriptor, - deps: &'a [model::FileDescriptor], -} - -impl<'a> Resolver<'a> { - fn map_entry_name_for_field_name(field_name: &str) -> String { - format!("{field_name}_MapEntry") - } - - fn map_entry_field( - &self, - name: &str, - number: i32, - field_type: &model::FieldType, - path_in_file: &RelativePath, - ) -> protobuf::descriptor::FieldDescriptorProto { - let mut output = protobuf::descriptor::FieldDescriptorProto::new(); - output.set_name(name.to_owned()); - output.set_number(number); - - let (t, t_name) = self.field_type(name, field_type, path_in_file); - output.set_type(t); - if let Some(t_name) = t_name { - output.set_type_name(t_name.path); - } - - output - } - - fn map_entry_message( - &self, - field_name: &str, - key: &model::FieldType, - value: &model::FieldType, - path_in_file: &RelativePath, - ) -> ConvertResult { - let mut output = protobuf::descriptor::DescriptorProto::new(); - - output.options.mut_or_insert_default().set_map_entry(true); - output.set_name(Resolver::map_entry_name_for_field_name(field_name)); - output - .field - .push(self.map_entry_field("key", 1, key, path_in_file)); - output - .field - .push(self.map_entry_field("value", 2, value, path_in_file)); - - Ok(output) - } - - fn message_options( - &self, - input: &[model::ProtobufOption], - ) -> ConvertResult { - let mut r = protobuf::descriptor::MessageOptions::new(); - self.custom_options( - input, - "google.protobuf.MessageOptions", - r.mut_unknown_fields(), - )?; - Ok(r) - } - - fn message( - &self, - input: &model::Message, - path_in_file: &RelativePath, - ) -> ConvertResult { - let nested_path_in_file = path_in_file.append(&input.name); - - let mut output = protobuf::descriptor::DescriptorProto::new(); - output.set_name(input.name.clone()); - - for m in &input.messages { - output - .nested_type - .push(self.message(m, &nested_path_in_file)?); - } - - for f in &input.fields { - if let model::FieldType::Map(ref t) = f.typ { - output.nested_type.push(self.map_entry_message( - &f.name, - &t.0, - &t.1, - path_in_file, - )?); - } - } - - output.enum_type = input - .enums - .iter() - .map(|e| self.enumeration(e)) - .collect::>()?; - - { - for f in &input.fields { - output - .field - .push(self.field(f, None, &nested_path_in_file)?); - } - - for (oneof_index, oneof) in input.oneofs.iter().enumerate() { - let oneof_index = oneof_index as i32; - for f in &oneof.fields { - output - .field - .push(self.field(f, Some(oneof_index), &nested_path_in_file)?); - } - } - } - - output.oneof_decl = input.oneofs.iter().map(|o| self.oneof(o)).collect(); - - *output.options.mut_or_insert_default() = self.message_options(&input.options)?; - - Ok(output) - } - - fn service_options( - &self, - input: &[model::ProtobufOption], - ) -> ConvertResult { - let mut r = protobuf::descriptor::ServiceOptions::new(); - self.custom_options( - input, - "google.protobuf.ServiceOptions", - r.mut_unknown_fields(), - )?; - Ok(r) - } - - fn method_options( - &self, - input: &[model::ProtobufOption], - ) -> ConvertResult { - let mut r = protobuf::descriptor::MethodOptions::new(); - self.custom_options( - input, - "google.protobuf.MethodOptions", - r.mut_unknown_fields(), - )?; - Ok(r) - } - - fn service( - &self, - input: &model::Service, - package: &str, - ) -> ConvertResult { - let mut output = protobuf::descriptor::ServiceDescriptorProto::new(); - output.set_name(input.name.clone()); - - for m in &input.methods { - let mut mm = protobuf::descriptor::MethodDescriptorProto::new(); - mm.set_name(m.name.clone()); - - mm.set_input_type(to_protobuf_absolute_path(package, m.input_type.clone())); - mm.set_output_type(to_protobuf_absolute_path(package, m.output_type.clone())); - - mm.set_client_streaming(m.client_streaming); - mm.set_server_streaming(m.server_streaming); - *mm.options.mut_or_insert_default() = self.method_options(&m.options)?; - - output.method.push(mm); - } - - *output.options.mut_or_insert_default() = self.service_options(&input.options)?; - - Ok(output) - } - - fn custom_options( - &self, - input: &[model::ProtobufOption], - extendee: &'static str, - unknown_fields: &mut protobuf::UnknownFields, - ) -> ConvertResult<()> { - for option in input { - // TODO: builtin options too - if !option.name.starts_with('(') { - continue; - } - - let extension = match self.find_extension(&option.name) { - Ok(e) => e, - // TODO: return error - Err(_) => continue, - }; - if extension.extendee != extendee { - return Err(ConvertError::WrongExtensionType( - option.name.clone(), - extendee, - )); - } - - let value = match Resolver::option_value_to_unknown_value( - &option.value, - &extension.field.typ, - &option.name, - ) { - Ok(value) => value, - Err(_) => { - // TODO: return error - continue; - } - }; - - unknown_fields.add_value(extension.field.number as u32, value); - } - Ok(()) - } - - fn field_options( - &self, - input: &[model::ProtobufOption], - ) -> ConvertResult { - let mut r = protobuf::descriptor::FieldOptions::new(); - if let Some(deprecated) = input.by_name_bool("deprecated")? { - r.set_deprecated(deprecated); - } - if let Some(packed) = input.by_name_bool("packed")? { - r.set_packed(packed); - } - self.custom_options( - input, - "google.protobuf.FieldOptions", - r.mut_unknown_fields(), - )?; - Ok(r) - } - - fn field( - &self, - input: &model::Field, - oneof_index: Option, - path_in_file: &RelativePath, - ) -> ConvertResult { - let mut output = protobuf::descriptor::FieldDescriptorProto::new(); - output.set_name(input.name.clone()); - - if let model::FieldType::Map(..) = input.typ { - output.set_label(protobuf::descriptor::field_descriptor_proto::Label::LABEL_REPEATED); - } else { - output.set_label(label(input.rule)); - } - - let (t, t_name) = self.field_type(&input.name, &input.typ, path_in_file); - output.set_type(t); - if let Some(t_name) = t_name { - output.set_type_name(t_name.path); - } - - output.set_number(input.number); - if let Some(default) = input.options.as_slice().by_name("default") { - let default = match output.type_() { - protobuf::descriptor::field_descriptor_proto::Type::TYPE_STRING => { - if let model::ProtobufConstant::String(ref s) = *default { - s.decode_utf8()? - } else { - return Err(ConvertError::DefaultValueIsNotStringLiteral); - } - } - protobuf::descriptor::field_descriptor_proto::Type::TYPE_BYTES => { - if let model::ProtobufConstant::String(ref s) = *default { - s.escaped.clone() - } else { - return Err(ConvertError::DefaultValueIsNotStringLiteral); - } - } - _ => default.format(), - }; - output.set_default_value(default); - } - - *output.options.mut_or_insert_default() = self.field_options(&input.options)?; - - if let Some(oneof_index) = oneof_index { - output.set_oneof_index(oneof_index); - } - - Ok(output) - } - - fn all_files(&self) -> Vec<&model::FileDescriptor> { - iter::once(self.current_file).chain(self.deps).collect() - } - - fn package_files(&self, package: &str) -> Vec<&model::FileDescriptor> { - self.all_files() - .into_iter() - .filter(|f| f.package == package) - .collect() - } - - fn current_file_package_files(&self) -> Vec<&model::FileDescriptor> { - self.package_files(&self.current_file.package) - } - - fn resolve_message_or_enum( - &self, - name: &str, - path_in_file: &RelativePath, - ) -> (AbsolutePath, MessageOrEnum) { - // find message or enum in current package - if !name.starts_with('.') { - for p in path_in_file.self_and_parents() { - let relative_path_with_name = p.clone(); - let relative_path_with_name = relative_path_with_name.append(name); - for file in self.current_file_package_files() { - if let Some((n, t)) = LookupScope::File(file).resolve_message_or_enum( - &AbsolutePath::from_path_without_dot(&file.package), - &relative_path_with_name, - ) { - return (n, t); - } - } - } - } - - // find message or enum in root package - { - let absolute_path = AbsolutePath::from_path_maybe_dot(name); - for file in self.all_files() { - let file_package = AbsolutePath::from_path_without_dot(&file.package); - if let Some(relative) = absolute_path.remove_prefix(&file_package) { - if let Some((n, t)) = - LookupScope::File(file).resolve_message_or_enum(&file_package, &relative) - { - return (n, t); - } - } - } - } - - panic!( - "couldn't find message or enum {} when parsing {}", - name, self.current_file.package - ); - } - - fn field_type( - &self, - name: &str, - input: &model::FieldType, - path_in_file: &RelativePath, - ) -> ( - protobuf::descriptor::field_descriptor_proto::Type, - Option, - ) { - match *input { - model::FieldType::Bool => ( - protobuf::descriptor::field_descriptor_proto::Type::TYPE_BOOL, - None, - ), - model::FieldType::Int32 => ( - protobuf::descriptor::field_descriptor_proto::Type::TYPE_INT32, - None, - ), - model::FieldType::Int64 => ( - protobuf::descriptor::field_descriptor_proto::Type::TYPE_INT64, - None, - ), - model::FieldType::Uint32 => ( - protobuf::descriptor::field_descriptor_proto::Type::TYPE_UINT32, - None, - ), - model::FieldType::Uint64 => ( - protobuf::descriptor::field_descriptor_proto::Type::TYPE_UINT64, - None, - ), - model::FieldType::Sint32 => ( - protobuf::descriptor::field_descriptor_proto::Type::TYPE_SINT32, - None, - ), - model::FieldType::Sint64 => ( - protobuf::descriptor::field_descriptor_proto::Type::TYPE_SINT64, - None, - ), - model::FieldType::Fixed32 => ( - protobuf::descriptor::field_descriptor_proto::Type::TYPE_FIXED32, - None, - ), - model::FieldType::Fixed64 => ( - protobuf::descriptor::field_descriptor_proto::Type::TYPE_FIXED64, - None, - ), - model::FieldType::Sfixed32 => ( - protobuf::descriptor::field_descriptor_proto::Type::TYPE_SFIXED32, - None, - ), - model::FieldType::Sfixed64 => ( - protobuf::descriptor::field_descriptor_proto::Type::TYPE_SFIXED64, - None, - ), - model::FieldType::Float => ( - protobuf::descriptor::field_descriptor_proto::Type::TYPE_FLOAT, - None, - ), - model::FieldType::Double => ( - protobuf::descriptor::field_descriptor_proto::Type::TYPE_DOUBLE, - None, - ), - model::FieldType::String => ( - protobuf::descriptor::field_descriptor_proto::Type::TYPE_STRING, - None, - ), - model::FieldType::Bytes => ( - protobuf::descriptor::field_descriptor_proto::Type::TYPE_BYTES, - None, - ), - model::FieldType::MessageOrEnum(ref name) => { - let (name, me) = self.resolve_message_or_enum(name, path_in_file); - (me.descriptor_type(), Some(name)) - } - model::FieldType::Map(..) => { - let mut type_name = AbsolutePath::from_path_without_dot(&self.current_file.package); - type_name.push_relative(path_in_file); - type_name.push_simple(&Resolver::map_entry_name_for_field_name(name)); - ( - protobuf::descriptor::field_descriptor_proto::Type::TYPE_MESSAGE, - Some(type_name), - ) - } - model::FieldType::Group(..) => ( - protobuf::descriptor::field_descriptor_proto::Type::TYPE_GROUP, - None, - ), - } - } - - fn enum_value( - &self, - name: &str, - number: i32, - ) -> protobuf::descriptor::EnumValueDescriptorProto { - let mut output = protobuf::descriptor::EnumValueDescriptorProto::new(); - output.set_name(name.to_owned()); - output.set_number(number); - output - } - - fn enum_options( - &self, - input: &[model::ProtobufOption], - ) -> ConvertResult { - let mut r = protobuf::descriptor::EnumOptions::new(); - if let Some(allow_alias) = input.by_name_bool("allow_alias")? { - r.set_allow_alias(allow_alias); - } - if let Some(deprecated) = input.by_name_bool("deprecated")? { - r.set_deprecated(deprecated); - } - self.custom_options(input, "google.protobuf.EnumOptions", r.mut_unknown_fields())?; - Ok(r) - } - - fn enumeration( - &self, - input: &model::Enumeration, - ) -> ConvertResult { - let mut output = protobuf::descriptor::EnumDescriptorProto::new(); - output.set_name(input.name.clone()); - output.value = input - .values - .iter() - .map(|v| self.enum_value(&v.name, v.number)) - .collect(); - *output.options.mut_or_insert_default() = self.enum_options(&input.options)?; - Ok(output) - } - - fn oneof(&self, input: &model::OneOf) -> protobuf::descriptor::OneofDescriptorProto { - let mut output = protobuf::descriptor::OneofDescriptorProto::new(); - output.set_name(input.name.clone()); - output - } - - fn find_extension_by_path(&self, path: &str) -> ConvertResult<&model::Extension> { - let (package, name) = match path.rfind('.') { - Some(dot) => (&path[..dot], &path[dot + 1..]), - None => (self.current_file.package.as_str(), path), - }; - - for file in self.package_files(package) { - for ext in &file.extensions { - if ext.field.name == name { - return Ok(ext); - } - } - } - - Err(ConvertError::ExtensionNotFound(path.to_owned())) - } - - fn find_extension(&self, option_name: &str) -> ConvertResult<&model::Extension> { - if !option_name.starts_with('(') || !option_name.ends_with(')') { - return Err(ConvertError::UnsupportedOption(option_name.to_owned())); - } - let path = &option_name[1..option_name.len() - 1]; - self.find_extension_by_path(path) - } - - fn option_value_to_unknown_value( - value: &model::ProtobufConstant, - field_type: &model::FieldType, - option_name: &str, - ) -> ConvertResult { - let v = match *value { - model::ProtobufConstant::Bool(b) => { - if field_type != &model::FieldType::Bool { - Err(()) - } else { - Ok(protobuf::UnknownValue::Varint(u64::from(b))) - } - } - // TODO: check overflow - model::ProtobufConstant::U64(v) => match *field_type { - model::FieldType::Fixed64 | model::FieldType::Sfixed64 => { - Ok(protobuf::UnknownValue::Fixed64(v)) - } - model::FieldType::Fixed32 | model::FieldType::Sfixed32 => { - Ok(protobuf::UnknownValue::Fixed32(v as u32)) - } - model::FieldType::Int64 - | model::FieldType::Int32 - | model::FieldType::Uint64 - | model::FieldType::Uint32 => Ok(protobuf::UnknownValue::Varint(v)), - model::FieldType::Sint64 => Ok(protobuf::UnknownValue::sint64(v as i64)), - model::FieldType::Sint32 => Ok(protobuf::UnknownValue::sint32(v as i32)), - _ => Err(()), - }, - model::ProtobufConstant::I64(v) => match *field_type { - model::FieldType::Fixed64 | model::FieldType::Sfixed64 => { - Ok(protobuf::UnknownValue::Fixed64(v as u64)) - } - model::FieldType::Fixed32 | model::FieldType::Sfixed32 => { - Ok(protobuf::UnknownValue::Fixed32(v as u32)) - } - model::FieldType::Int64 - | model::FieldType::Int32 - | model::FieldType::Uint64 - | model::FieldType::Uint32 => Ok(protobuf::UnknownValue::Varint(v as u64)), - model::FieldType::Sint64 => Ok(protobuf::UnknownValue::sint64(v)), - model::FieldType::Sint32 => Ok(protobuf::UnknownValue::sint32(v as i32)), - _ => Err(()), - }, - model::ProtobufConstant::F64(f) => match *field_type { - model::FieldType::Float => { - Ok(protobuf::UnknownValue::Fixed32((f as f32).to_bits())) - } - model::FieldType::Double => Ok(protobuf::UnknownValue::Fixed64(f.to_bits())), - _ => Err(()), - }, - model::ProtobufConstant::String(ref s) => { - match *field_type { - model::FieldType::String => Ok(protobuf::UnknownValue::LengthDelimited( - s.decode_utf8()?.into_bytes(), - )), - // TODO: bytes - _ => Err(()), - } - } - _ => Err(()), - }; - - v.map_err(|()| { - ConvertError::UnsupportedExtensionType( - option_name.to_owned(), - format!("{field_type:?}"), - ) - }) - } - - fn file_options( - &self, - input: &[model::ProtobufOption], - ) -> ConvertResult { - let mut r = protobuf::descriptor::FileOptions::new(); - self.custom_options(input, "google.protobuf.FileOptions", r.mut_unknown_fields())?; - Ok(r) - } - - fn extension( - &self, - input: &model::Extension, - ) -> ConvertResult { - let relative_path = RelativePath::new("".to_owned()); - let mut field = self.field(&input.field, None, &relative_path)?; - field.set_extendee( - self.resolve_message_or_enum(&input.extendee, &relative_path) - .0 - .path, - ); - Ok(field) - } -} - -fn to_protobuf_absolute_path(package: &str, path: String) -> String { - if !path.starts_with('.') { - if path.contains('.') { - return format!(".{}", &path); - } else { - return format!(".{}.{}", package, &path); - } - } - - path -} - -fn syntax(input: model::Syntax) -> String { - match input { - model::Syntax::Proto2 => "proto2".to_owned(), - model::Syntax::Proto3 => "proto3".to_owned(), - } -} - -fn label(input: model::Rule) -> protobuf::descriptor::field_descriptor_proto::Label { - match input { - model::Rule::Optional => { - protobuf::descriptor::field_descriptor_proto::Label::LABEL_OPTIONAL - } - model::Rule::Required => { - protobuf::descriptor::field_descriptor_proto::Label::LABEL_REQUIRED - } - model::Rule::Repeated => { - protobuf::descriptor::field_descriptor_proto::Label::LABEL_REPEATED - } - } -} - -pub fn file_descriptor( - name: String, - input: &model::FileDescriptor, - deps: &[model::FileDescriptor], -) -> ConvertResult { - let resolver = Resolver { - current_file: input, - deps, - }; - - let mut output = protobuf::descriptor::FileDescriptorProto::new(); - output.set_name(name); - output.set_package(input.package.clone()); - output.set_syntax(syntax(input.syntax)); - - for m in &input.messages { - output - .message_type - .push(resolver.message(m, &RelativePath::empty())?); - } - - for s in &input.services { - output.service.push(resolver.service(s, &input.package)?); - } - - output.enum_type = input - .enums - .iter() - .map(|e| resolver.enumeration(e)) - .collect::>()?; - - *output.options.mut_or_insert_default() = resolver.file_options(&input.options)?; - - for e in &input.extensions { - output.extension.push(resolver.extension(e)?); - } - - Ok(output) -} diff --git a/ttrpc-codegen/src/lib.rs b/ttrpc-codegen/src/lib.rs index 5f2d2f87..abf6c4c7 100644 --- a/ttrpc-codegen/src/lib.rs +++ b/ttrpc-codegen/src/lib.rs @@ -1,4 +1,3 @@ -#![allow(dead_code)] //! API to generate .rs files for ttrpc from protobuf //! //! @@ -31,21 +30,11 @@ pub use protobuf_codegen::{ Customize as ProtobufCustomize, CustomizeCallback as ProtobufCustomizeCallback, }; -use std::collections::HashMap; -use std::error::Error; -use std::fmt; -use std::fs; use std::io; -use std::io::Read; use std::path::Path; use std::path::PathBuf; pub use ttrpc_compiler::Customize; -mod convert; -mod model; -mod parser; -mod str_lit; - /// Invoke pure rust codegen. #[derive(Debug, Default)] pub struct Codegen { @@ -76,6 +65,9 @@ impl Codegen { } /// Add an include directory. + /// + /// Canonical Google well-known type imports are resolved automatically and + /// do not need to be present in an include directory. pub fn include(&mut self, include: impl AsRef) -> &mut Self { self.includes.push(include.as_ref().to_owned()); self @@ -164,175 +156,6 @@ impl Codegen { } } -/// Convert OS path to protobuf path (with slashes) -/// Function is `pub(crate)` for test. -pub(crate) fn relative_path_to_protobuf_path(path: &Path) -> String { - assert!(path.is_relative()); - let path = path.to_str().expect("not a valid UTF-8 name"); - if cfg!(windows) { - path.replace('\\', "/") - } else { - path.to_owned() - } -} - -#[derive(Clone)] -struct FileDescriptorPair { - parsed: model::FileDescriptor, - descriptor: protobuf::descriptor::FileDescriptorProto, -} - -#[derive(Debug)] -enum CodegenError { - ParserErrorWithLocation(parser::ParserErrorWithLocation), - ConvertError(convert::ConvertError), -} - -impl From for CodegenError { - fn from(e: parser::ParserErrorWithLocation) -> Self { - CodegenError::ParserErrorWithLocation(e) - } -} - -impl From for CodegenError { - fn from(e: convert::ConvertError) -> Self { - CodegenError::ConvertError(e) - } -} - -#[derive(Debug)] -struct WithFileError { - file: String, - error: CodegenError, -} - -impl fmt::Display for WithFileError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!( - f, - "WithFileError(file: {:?}, error: {:?})", - &self.file, &self.error - ) - } -} - -impl Error for WithFileError { - fn description(&self) -> &str { - "WithFileError" - } -} - -struct Run<'a> { - parsed_files: HashMap, - includes: &'a [&'a Path], -} - -impl<'a> Run<'a> { - fn get_file_and_all_deps_already_parsed( - &self, - protobuf_path: &str, - result: &mut HashMap, - ) { - if result.contains_key(protobuf_path) { - return; - } - - let pair = self - .parsed_files - .get(protobuf_path) - .expect("must be already parsed"); - result.insert(protobuf_path.to_owned(), pair.clone()); - - self.get_all_deps_already_parsed(&pair.parsed, result); - } - - fn get_all_deps_already_parsed( - &self, - parsed: &model::FileDescriptor, - result: &mut HashMap, - ) { - for import in &parsed.import_paths { - self.get_file_and_all_deps_already_parsed(import, result); - } - } - - fn add_file(&mut self, protobuf_path: &str, fs_path: &Path) -> io::Result<()> { - if self.parsed_files.contains_key(protobuf_path) { - return Ok(()); - } - - let mut content = String::new(); - fs::File::open(fs_path)?.read_to_string(&mut content)?; - - let parsed = model::FileDescriptor::parse(content).map_err(|e| { - io::Error::other(WithFileError { - file: format!("{}", fs_path.display()), - error: e.into(), - }) - })?; - - for import_path in &parsed.import_paths { - self.add_imported_file(import_path)?; - } - - let mut this_file_deps = HashMap::new(); - self.get_all_deps_already_parsed(&parsed, &mut this_file_deps); - - let this_file_deps: Vec<_> = this_file_deps.into_values().map(|v| v.parsed).collect(); - - let descriptor = - convert::file_descriptor(protobuf_path.to_owned(), &parsed, &this_file_deps).map_err( - |e| { - io::Error::other(WithFileError { - file: format!("{}", fs_path.display()), - error: e.into(), - }) - }, - )?; - - self.parsed_files.insert( - protobuf_path.to_owned(), - FileDescriptorPair { parsed, descriptor }, - ); - - Ok(()) - } - - fn add_imported_file(&mut self, protobuf_path: &str) -> io::Result<()> { - for include_dir in self.includes { - let fs_path = Path::new(include_dir).join(protobuf_path); - if fs_path.exists() { - return self.add_file(protobuf_path, &fs_path); - } - } - - Err(io::Error::other(format!( - "protobuf path {:?} is not found in import path {:?}", - protobuf_path, self.includes - ))) - } - - fn add_fs_file(&mut self, fs_path: &Path) -> io::Result { - let relative_path = self - .includes - .iter() - .filter_map(|include_dir| fs_path.strip_prefix(include_dir).ok()) - .next(); - - match relative_path { - Some(relative_path) => { - let protobuf_path = relative_path_to_protobuf_path(relative_path); - self.add_file(&protobuf_path, fs_path)?; - Ok(protobuf_path) - } - None => Err(io::Error::other(format!( - "file {:?} must reside in include path {:?}", - fs_path, self.includes - ))), - } - } -} - #[doc(hidden)] pub struct ParsedAndTypechecked { pub relative_paths: Vec, @@ -344,47 +167,22 @@ pub fn parse_and_typecheck( includes: &[&Path], input: &[&Path], ) -> io::Result { - let mut run = Run { - parsed_files: HashMap::new(), - includes, - }; + let mut parser = protobuf_parse::Parser::new(); + parser + .pure() + .includes(includes.iter().copied()) + .inputs(input.iter().copied()); - let mut relative_paths = Vec::new(); - - for input in input { - relative_paths.push(run.add_fs_file(Path::new(input))?); - } - - let file_descriptors: Vec<_> = run - .parsed_files - .into_values() - .map(|v| v.descriptor) - .collect(); + let parsed = parser + .parse_and_typecheck() + .map_err(|error| io::Error::other(format!("{error:#}")))?; Ok(ParsedAndTypechecked { - relative_paths, - file_descriptors, + relative_paths: parsed + .relative_paths + .into_iter() + .map(|path| path.to_string()) + .collect(), + file_descriptors: parsed.file_descriptors, }) } - -#[cfg(test)] -mod test { - use super::*; - - #[cfg(windows)] - #[test] - fn test_relative_path_to_protobuf_path_windows() { - assert_eq!( - "foo/bar.proto", - relative_path_to_protobuf_path(Path::new("foo\\bar.proto")) - ); - } - - #[test] - fn test_relative_path_to_protobuf_path() { - assert_eq!( - "foo/bar.proto", - relative_path_to_protobuf_path(Path::new("foo/bar.proto")) - ); - } -} diff --git a/ttrpc-codegen/src/model.rs b/ttrpc-codegen/src/model.rs deleted file mode 100644 index b60bf72e..00000000 --- a/ttrpc-codegen/src/model.rs +++ /dev/null @@ -1,299 +0,0 @@ -//! A nom-based protobuf file parser -//! -//! This crate can be seen as a rust transcription of the -//! [descriptor.proto](https://github.com/google/protobuf/blob/master/src/google/protobuf/descriptor.proto) file - -use crate::parser::{Loc, Parser, ParserErrorWithLocation}; -use crate::str_lit::StrLit; -use protobuf_support::lexer::float; - -/// Protobox syntax -#[derive(Debug, Clone, Copy, Eq, PartialEq, Default)] -pub enum Syntax { - /// Protobuf syntax [2](https://developers.google.com/protocol-buffers/docs/proto) (default) - #[default] - Proto2, - /// Protobuf syntax [3](https://developers.google.com/protocol-buffers/docs/proto3) - Proto3, -} - -/// A field rule -#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] -pub enum Rule { - /// A well-formed message can have zero or one of this field (but not more than one). - Optional, - /// This field can be repeated any number of times (including zero) in a well-formed message. - /// The order of the repeated values will be preserved. - Repeated, - /// A well-formed message must have exactly one of this field. - Required, -} - -/// Protobuf supported field types -#[derive(Debug, Clone, PartialEq)] -pub enum FieldType { - /// Protobuf int32 - /// - /// # Remarks - /// - /// Uses variable-length encoding. Inefficient for encoding negative numbers – if - /// your field is likely to have negative values, use sint32 instead. - Int32, - /// Protobuf int64 - /// - /// # Remarks - /// - /// Uses variable-length encoding. Inefficient for encoding negative numbers – if - /// your field is likely to have negative values, use sint64 instead. - Int64, - /// Protobuf uint32 - /// - /// # Remarks - /// - /// Uses variable-length encoding. - Uint32, - /// Protobuf uint64 - /// - /// # Remarks - /// - /// Uses variable-length encoding. - Uint64, - /// Protobuf sint32 - /// - /// # Remarks - /// - /// Uses ZigZag variable-length encoding. Signed int value. These more efficiently - /// encode negative numbers than regular int32s. - Sint32, - /// Protobuf sint64 - /// - /// # Remarks - /// - /// Uses ZigZag variable-length encoding. Signed int value. These more efficiently - /// encode negative numbers than regular int32s. - Sint64, - /// Protobuf bool - Bool, - /// Protobuf fixed64 - /// - /// # Remarks - /// - /// Always eight bytes. More efficient than uint64 if values are often greater than 2^56. - Fixed64, - /// Protobuf sfixed64 - /// - /// # Remarks - /// - /// Always eight bytes. - Sfixed64, - /// Protobuf double - Double, - /// Protobuf string - /// - /// # Remarks - /// - /// A string must always contain UTF-8 encoded or 7-bit ASCII text. - String, - /// Protobuf bytes - /// - /// # Remarks - /// - /// May contain any arbitrary sequence of bytes. - Bytes, - /// Protobut fixed32 - /// - /// # Remarks - /// - /// Always four bytes. More efficient than uint32 if values are often greater than 2^28. - Fixed32, - /// Protobut sfixed32 - /// - /// # Remarks - /// - /// Always four bytes. - Sfixed32, - /// Protobut float - Float, - /// Protobuf message or enum (holds the name) - MessageOrEnum(String), - /// Protobut map - Map(Box<(FieldType, FieldType)>), - /// Protobuf group (deprecated) - Group(Vec), -} - -/// A Protobuf Field -#[derive(Debug, Clone, PartialEq)] -pub struct Field { - /// Field name - pub name: String, - /// Field `Rule` - pub rule: Rule, - /// Field type - pub typ: FieldType, - /// Tag number - pub number: i32, - /// Non-builtin options - pub options: Vec, -} - -/// Extension range -#[derive(Default, Debug, Eq, PartialEq, Copy, Clone)] -pub struct FieldNumberRange { - /// First number - pub from: i32, - /// Inclusive - pub to: i32, -} - -/// A protobuf message -#[derive(Debug, Clone, Default)] -pub struct Message { - /// Message name - pub name: String, - /// Message `Field`s - pub fields: Vec, - /// Message `OneOf`s - pub oneofs: Vec, - /// Message reserved numbers - /// - /// TODO: use RangeInclusive once stable - pub reserved_nums: Vec, - /// Message reserved names - pub reserved_names: Vec, - /// Nested messages - pub messages: Vec, - /// Nested enums - pub enums: Vec, - /// Non-builtin options - pub options: Vec, -} - -/// A protobuf enumeration field -#[derive(Debug, Clone)] -pub struct EnumValue { - /// enum value name - pub name: String, - /// enum value number - pub number: i32, -} - -/// A protobuf enumerator -#[derive(Debug, Clone)] -pub struct Enumeration { - /// enum name - pub name: String, - /// enum values - pub values: Vec, - /// enum options - pub options: Vec, -} - -/// A OneOf -#[derive(Debug, Clone, Default)] -pub struct OneOf { - /// OneOf name - pub name: String, - /// OneOf fields - pub fields: Vec, -} - -#[derive(Debug, Clone)] -pub struct Extension { - /// Extend this type with field - pub extendee: String, - /// Extension field - pub field: Field, -} - -/// Service method -#[derive(Debug, Clone)] -pub struct Method { - /// Method name - pub name: String, - /// Input type - pub input_type: String, - /// Output type - pub output_type: String, - /// If this method is client streaming - pub client_streaming: bool, - /// If this method is server streaming - pub server_streaming: bool, - /// Method options - pub options: Vec, -} - -/// Service definition -#[derive(Debug, Clone)] -pub struct Service { - /// Service name - pub name: String, - pub methods: Vec, - pub options: Vec, -} - -#[derive(Debug, Clone, PartialEq)] -pub enum ProtobufConstant { - U64(u64), - I64(i64), - F64(f64), // TODO: eq - Bool(bool), - Ident(String), - String(StrLit), - BracedExpr(String), -} - -impl ProtobufConstant { - pub fn format(&self) -> String { - match *self { - ProtobufConstant::U64(u) => u.to_string(), - ProtobufConstant::I64(i) => i.to_string(), - ProtobufConstant::F64(f) => float::format_protobuf_float(f), - ProtobufConstant::Bool(b) => b.to_string(), - ProtobufConstant::Ident(ref i) => i.clone(), - ProtobufConstant::String(ref s) => s.quoted(), - ProtobufConstant::BracedExpr(ref s) => s.clone(), - } - } -} - -#[derive(Debug, Clone, PartialEq)] -pub struct ProtobufOption { - pub name: String, - pub value: ProtobufConstant, -} - -/// A File descriptor representing a whole .proto file -#[derive(Debug, Default, Clone)] -pub struct FileDescriptor { - /// Imports - pub import_paths: Vec, - /// Package - pub package: String, - /// Protobuf Syntax - pub syntax: Syntax, - /// Top level messages - pub messages: Vec, - /// Enums - pub enums: Vec, - /// Extensions - pub extensions: Vec, - /// Services - pub services: Vec, - /// Non-builtin options - pub options: Vec, -} - -impl FileDescriptor { - /// Parses a .proto file content into a `FileDescriptor` - pub fn parse>(file: S) -> Result { - let mut parser = Parser::new(file.as_ref()); - match parser.next_proto() { - Ok(r) => Ok(r), - Err(error) => { - let Loc { line, col } = parser.loc(); - Err(ParserErrorWithLocation { error, line, col }) - } - } - } -} diff --git a/ttrpc-codegen/src/parser.rs b/ttrpc-codegen/src/parser.rs deleted file mode 100644 index 0c39cb10..00000000 --- a/ttrpc-codegen/src/parser.rs +++ /dev/null @@ -1,2212 +0,0 @@ -use std::f64; -use std::fmt; -use std::num::ParseIntError; -use std::str; - -use crate::model::*; -use crate::str_lit::*; -use protobuf_support::lexer::float; - -const FIRST_LINE: u32 = 1; -const FIRST_COL: u32 = 1; - -/// Location in file -#[derive(Copy, Clone, Debug, Eq, PartialEq)] -pub struct Loc { - /// 1-based - pub line: u32, - /// 1-based - pub col: u32, -} - -impl fmt::Display for Loc { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{}:{}", self.line, self.col) - } -} - -impl Loc { - pub fn start() -> Loc { - Loc { - line: FIRST_LINE, - col: FIRST_COL, - } - } -} - -/// Basic information about parsing error. -#[derive(Debug)] -pub enum ParserError { - IncorrectInput, - IncorrectFloatLit, - NotUtf8, - ExpectChar(char), - ExpectConstant, - ExpectIdent, - ExpectHexDigit, - ExpectOctDigit, - ExpectDecDigit, - UnknownSyntax, - UnexpectedEof, - ParseIntError, - IntegerOverflow, - LabelNotAllowed, - LabelRequired, - InternalError, - StrLitDecodeError(StrLitDecodeError), - GroupNameShouldStartWithUpperCase, - MapFieldNotAllowed, - ExpectNamedIdent(String), -} - -#[derive(Debug)] -pub struct ParserErrorWithLocation { - pub error: ParserError, - /// 1-based - pub line: u32, - /// 1-based - pub col: u32, -} - -impl From for ParserError { - fn from(e: StrLitDecodeError) -> Self { - ParserError::StrLitDecodeError(e) - } -} - -impl From for ParserError { - fn from(_: ParseIntError) -> Self { - ParserError::ParseIntError - } -} - -impl From for ParserError { - fn from(_: float::ProtobufFloatParseError) -> Self { - ParserError::IncorrectFloatLit - } -} - -pub type ParserResult = Result; - -trait ToU8 { - fn to_u8(&self) -> ParserResult; -} - -trait ToI32 { - fn to_i32(&self) -> ParserResult; -} - -trait ToI64 { - fn to_i64(&self) -> ParserResult; -} - -trait ToChar { - fn to_char(&self) -> ParserResult; -} - -impl ToI32 for u64 { - fn to_i32(&self) -> ParserResult { - if *self <= i32::MAX as u64 { - Ok(*self as i32) - } else { - Err(ParserError::IntegerOverflow) - } - } -} - -impl ToI32 for i64 { - fn to_i32(&self) -> ParserResult { - if *self <= i32::MAX as i64 && *self >= i32::MIN as i64 { - Ok(*self as i32) - } else { - Err(ParserError::IntegerOverflow) - } - } -} - -impl ToI64 for u64 { - fn to_i64(&self) -> Result { - if *self <= i64::MAX as u64 { - Ok(*self as i64) - } else { - Err(ParserError::IntegerOverflow) - } - } -} - -impl ToChar for u8 { - fn to_char(&self) -> Result { - if *self <= 0x7f { - Ok(*self as char) - } else { - Err(ParserError::NotUtf8) - } - } -} - -impl ToU8 for u32 { - fn to_u8(&self) -> Result { - if *self as u8 as u32 == *self { - Ok(*self as u8) - } else { - Err(ParserError::IntegerOverflow) - } - } -} - -trait U64Extensions { - fn neg(&self) -> ParserResult; -} - -impl U64Extensions for u64 { - fn neg(&self) -> ParserResult { - if *self <= 0x7fff_ffff_ffff_ffff { - Ok(-(*self as i64)) - } else if *self == 0x8000_0000_0000_0000 { - Ok(-0x8000_0000_0000_0000) - } else { - Err(ParserError::IntegerOverflow) - } - } -} - -#[derive(Clone, Debug, PartialEq)] -enum Token { - Ident(String), - Symbol(char), - IntLit(u64), - // including quotes - StrLit(StrLit), - FloatLit(f64), -} - -impl Token { - /// Back to original - fn format(&self) -> String { - match *self { - Token::Ident(ref s) => s.clone(), - Token::Symbol(c) => c.to_string(), - Token::IntLit(ref i) => i.to_string(), - Token::StrLit(ref s) => s.quoted(), - Token::FloatLit(ref f) => f.to_string(), - } - } - - fn to_num_lit(&self) -> ParserResult { - match *self { - Token::IntLit(i) => Ok(NumLit::U64(i)), - Token::FloatLit(f) => Ok(NumLit::F64(f)), - _ => Err(ParserError::IncorrectInput), - } - } -} - -#[derive(Clone)] -struct TokenWithLocation { - token: Token, - loc: Loc, -} - -#[derive(Copy, Clone)] -pub struct Lexer<'a> { - pub input: &'a str, - pub pos: usize, - pub loc: Loc, -} - -fn is_letter(c: char) -> bool { - c.is_alphabetic() || c == '_' -} - -impl<'a> Lexer<'a> { - /// No more chars - pub fn eof(&self) -> bool { - self.pos == self.input.len() - } - - /// Remaining chars - fn rem_chars(&self) -> &'a str { - &self.input[self.pos..] - } - - fn lookahead_char_is_in(&self, alphabet: &str) -> bool { - self.lookahead_char().is_some_and(|c| alphabet.contains(c)) - } - - fn next_char_opt(&mut self) -> Option { - let rem = self.rem_chars(); - if rem.is_empty() { - None - } else { - let mut char_indices = rem.char_indices(); - let (_, c) = char_indices.next().unwrap(); - let c_len = char_indices.next().map(|(len, _)| len).unwrap_or(rem.len()); - self.pos += c_len; - if c == '\n' { - self.loc.line += 1; - self.loc.col = FIRST_COL; - } else { - self.loc.col += 1; - } - Some(c) - } - } - - fn next_char(&mut self) -> ParserResult { - self.next_char_opt().ok_or(ParserError::UnexpectedEof) - } - - /// Skip whitespaces - fn skip_whitespaces(&mut self) { - self.take_while(|c| c.is_whitespace()); - } - - fn skip_comment(&mut self) -> ParserResult<()> { - if self.skip_if_lookahead_is_str("/*") { - let end = "*/"; - match self.rem_chars().find(end) { - None => Err(ParserError::UnexpectedEof), - Some(len) => { - let new_pos = self.pos + len + end.len(); - self.skip_to_pos(new_pos); - Ok(()) - } - } - } else { - Ok(()) - } - } - - fn skip_block_comment(&mut self) { - if self.skip_if_lookahead_is_str("//") { - loop { - match self.next_char_opt() { - Some('\n') | None => break, - _ => {} - } - } - } - } - - fn skip_ws(&mut self) -> ParserResult<()> { - loop { - let pos = self.pos; - self.skip_whitespaces(); - self.skip_comment()?; - self.skip_block_comment(); - if pos == self.pos { - // Did not advance - return Ok(()); - } - } - } - - fn take_while(&mut self, f: F) -> &'a str - where - F: Fn(char) -> bool, - { - let start = self.pos; - while self.lookahead_char().map(&f) == Some(true) { - self.next_char_opt().unwrap(); - } - let end = self.pos; - &self.input[start..end] - } - - fn lookahead_char(&self) -> Option { - self.clone().next_char_opt() - } - - fn lookahead_is_str(&self, s: &str) -> bool { - self.rem_chars().starts_with(s) - } - - fn skip_if_lookahead_is_str(&mut self, s: &str) -> bool { - if self.lookahead_is_str(s) { - let new_pos = self.pos + s.len(); - self.skip_to_pos(new_pos); - true - } else { - false - } - } - - fn next_char_if

(&mut self, p: P) -> Option - where - P: FnOnce(char) -> bool, - { - let mut clone = *self; - match clone.next_char_opt() { - Some(c) if p(c) => { - *self = clone; - Some(c) - } - _ => None, - } - } - - fn next_char_if_eq(&mut self, expect: char) -> bool { - self.next_char_if(|c| c == expect).is_some() - } - - fn next_char_if_in(&mut self, alphabet: &str) -> Option { - alphabet.chars().find(|&c| self.next_char_if_eq(c)) - } - - fn next_char_expect_eq(&mut self, expect: char) -> ParserResult<()> { - if self.next_char_if_eq(expect) { - Ok(()) - } else { - Err(ParserError::ExpectChar(expect)) - } - } - - // str functions - - /// properly update line and column - fn skip_to_pos(&mut self, new_pos: usize) -> &'a str { - assert!(new_pos >= self.pos); - assert!(new_pos <= self.input.len()); - let pos = self.pos; - while self.pos != new_pos { - self.next_char_opt().unwrap(); - } - &self.input[pos..new_pos] - } - - // Protobuf grammar - - // char functions - - // letter = "A" … "Z" | "a" … "z" - // https://github.com/google/protobuf/issues/4565 - fn next_letter_opt(&mut self) -> Option { - self.next_char_if(is_letter) - } - - // capitalLetter = "A" … "Z" - fn _next_capital_letter_opt(&mut self) -> Option { - self.next_char_if(|c| c.is_ascii_uppercase()) - } - - fn is_ascii_alphanumeric(c: char) -> bool { - c.is_ascii_lowercase() || c.is_ascii_uppercase() || c.is_ascii_digit() - } - - fn next_ident_part(&mut self) -> Option { - self.next_char_if(|c| Lexer::is_ascii_alphanumeric(c) || c == '_') - } - - // Identifiers - - // ident = letter { letter | decimalDigit | "_" } - fn next_ident_opt(&mut self) -> ParserResult> { - if let Some(c) = self.next_letter_opt() { - let mut ident = String::new(); - ident.push(c); - while let Some(c) = self.next_ident_part() { - ident.push(c); - } - Ok(Some(ident)) - } else { - Ok(None) - } - } - - // Integer literals - - fn is_ascii_hexdigit(c: char) -> bool { - c.is_ascii_digit() || ('a'..='f').contains(&c) || ('A'..='F').contains(&c) - } - - // hexLit = "0" ( "x" | "X" ) hexDigit { hexDigit } - fn next_hex_lit(&mut self) -> ParserResult> { - Ok( - if self.skip_if_lookahead_is_str("0x") || self.skip_if_lookahead_is_str("0X") { - let s = self.take_while(Lexer::is_ascii_hexdigit); - Some(u64::from_str_radix(s, 16)?) - } else { - None - }, - ) - } - - fn is_ascii_digit(c: char) -> bool { - c.is_ascii_digit() - } - - // decimalLit = ( "1" … "9" ) { decimalDigit } - // octalLit = "0" { octalDigit } - fn next_decimal_octal_lit(&mut self) -> ParserResult> { - // do not advance on number parse error - let mut clone = *self; - - let pos = clone.pos; - - Ok(if clone.next_char_if(Lexer::is_ascii_digit).is_some() { - clone.take_while(Lexer::is_ascii_digit); - let value = clone.input[pos..clone.pos].parse()?; - *self = clone; - Some(value) - } else { - None - }) - } - - // hexDigit = "0" … "9" | "A" … "F" | "a" … "f" - fn next_hex_digit(&mut self) -> ParserResult { - let mut clone = *self; - let r = match clone.next_char()? { - c if c.is_ascii_digit() => c as u32 - b'0' as u32, - c if ('A'..='F').contains(&c) => c as u32 - b'A' as u32 + 10, - c if ('a'..='f').contains(&c) => c as u32 - b'a' as u32 + 10, - _ => return Err(ParserError::ExpectHexDigit), - }; - *self = clone; - Ok(r) - } - - // octalDigit = "0" … "7" - fn next_octal_digit(&mut self) -> ParserResult { - let mut clone = *self; - let r = match clone.next_char()? { - c if ('0'..='7').contains(&c) => c as u32 - b'0' as u32, - _ => return Err(ParserError::ExpectOctDigit), - }; - *self = clone; - Ok(r) - } - - // decimalDigit = "0" … "9" - fn next_decimal_digit(&mut self) -> ParserResult { - let mut clone = *self; - let r = match clone.next_char()? { - c if c.is_ascii_digit() => c as u32 - '0' as u32, - _ => return Err(ParserError::ExpectDecDigit), - }; - *self = clone; - Ok(r) - } - - // decimals = decimalDigit { decimalDigit } - fn next_decimal_digits(&mut self) -> ParserResult<()> { - self.next_decimal_digit()?; - self.take_while(|c| c.is_ascii_digit()); - Ok(()) - } - - // intLit = decimalLit | octalLit | hexLit - fn next_int_lit_opt(&mut self) -> ParserResult> { - self.skip_ws()?; - if let Some(i) = self.next_hex_lit()? { - return Ok(Some(i)); - } - if let Some(i) = self.next_decimal_octal_lit()? { - return Ok(Some(i)); - } - Ok(None) - } - - // Floating-point literals - - // exponent = ( "e" | "E" ) [ "+" | "-" ] decimals - fn next_exponent_opt(&mut self) -> ParserResult> { - if self.next_char_if_in("eE").is_some() { - self.next_char_if_in("+-"); - self.next_decimal_digits()?; - Ok(Some(())) - } else { - Ok(None) - } - } - - // floatLit = ( decimals "." [ decimals ] [ exponent ] | decimals exponent | "."decimals [ exponent ] ) | "inf" | "nan" - fn next_float_lit(&mut self) -> ParserResult<()> { - // "inf" and "nan" are handled as part of ident - if self.next_char_if_eq('.') { - self.next_decimal_digits()?; - self.next_exponent_opt()?; - } else { - self.next_decimal_digits()?; - if self.next_char_if_eq('.') { - self.next_decimal_digits()?; - self.next_exponent_opt()?; - } else if (self.next_exponent_opt()?).is_none() { - return Err(ParserError::IncorrectFloatLit); - } - } - Ok(()) - } - - // String literals - - // charValue = hexEscape | octEscape | charEscape | /[^\0\n\\]/ - // hexEscape = '\' ( "x" | "X" ) hexDigit hexDigit - // https://github.com/google/protobuf/issues/4560 - // octEscape = '\' octalDigit octalDigit octalDigit - // charEscape = '\' ( "a" | "b" | "f" | "n" | "r" | "t" | "v" | '\' | "'" | '"' ) - // quote = "'" | '"' - pub fn next_char_value(&mut self) -> ParserResult { - match self.next_char()? { - '\\' => { - match self.next_char()? { - '\'' => Ok('\''), - '"' => Ok('"'), - '\\' => Ok('\\'), - 'a' => Ok('\x07'), - 'b' => Ok('\x08'), - 'f' => Ok('\x0c'), - 'n' => Ok('\n'), - 'r' => Ok('\r'), - 't' => Ok('\t'), - 'v' => Ok('\x0b'), - 'x' => { - let d1 = self.next_hex_digit()? as u8; - let d2 = self.next_hex_digit()? as u8; - // TODO: do not decode as char if > 0x80 - Ok(((d1 << 4) | d2) as char) - } - d if ('0'..='7').contains(&d) => { - let mut r = d as u8 - b'0'; - for _ in 0..2 { - match self.next_octal_digit() { - Err(_) => break, - Ok(d) => r = (r << 3) + d as u8, - } - } - // TODO: do not decode as char if > 0x80 - Ok(r as char) - } - // https://github.com/google/protobuf/issues/4562 - c => Ok(c), - } - } - '\n' | '\0' => Err(ParserError::IncorrectInput), - c => Ok(c), - } - } - - // https://github.com/google/protobuf/issues/4564 - // strLit = ( "'" { charValue } "'" ) | ( '"' { charValue } '"' ) - fn next_str_lit_raw(&mut self) -> ParserResult { - let mut raw = String::new(); - - let mut first = true; - loop { - if !first { - self.skip_ws()?; - } - - let start = self.pos; - - let q = match self.next_char_if_in("'\"") { - Some(q) => q, - None if !first => break, - None => return Err(ParserError::IncorrectInput), - }; - first = false; - while self.lookahead_char() != Some(q) { - self.next_char_value()?; - } - self.next_char_expect_eq(q)?; - - raw.push_str(&self.input[start + 1..self.pos - 1]); - } - Ok(raw) - } - - fn next_str_lit_raw_opt(&mut self) -> ParserResult> { - if self.lookahead_char_is_in("'\"") { - Ok(Some(self.next_str_lit_raw()?)) - } else { - Ok(None) - } - } - - fn is_ascii_punctuation(c: char) -> bool { - matches!( - c, - '.' | ',' - | ':' - | ';' - | '/' - | '\\' - | '=' - | '%' - | '+' - | '-' - | '*' - | '<' - | '>' - | '(' - | ')' - | '{' - | '}' - | '[' - | ']' - ) - } - - fn next_token_inner(&mut self) -> ParserResult { - if let Some(ident) = self.next_ident_opt()? { - let token = if ident == float::PROTOBUF_NAN { - Token::FloatLit(f64::NAN) - } else if ident == float::PROTOBUF_INF { - Token::FloatLit(f64::INFINITY) - } else { - Token::Ident(ident) - }; - return Ok(token); - } - - let mut clone = *self; - let pos = clone.pos; - if clone.next_float_lit().is_ok() { - let f = float::parse_protobuf_float(&self.input[pos..clone.pos])?; - *self = clone; - return Ok(Token::FloatLit(f)); - } - - if let Some(lit) = self.next_int_lit_opt()? { - return Ok(Token::IntLit(lit)); - } - - if let Some(escaped) = self.next_str_lit_raw_opt()? { - return Ok(Token::StrLit(StrLit { escaped })); - } - - // This branch must be after str lit - if let Some(c) = self.next_char_if(Lexer::is_ascii_punctuation) { - return Ok(Token::Symbol(c)); - } - - if let Some(ident) = self.next_ident_opt()? { - return Ok(Token::Ident(ident)); - } - - Err(ParserError::IncorrectInput) - } - - fn next_token(&mut self) -> ParserResult> { - self.skip_ws()?; - let loc = self.loc; - - Ok(if self.eof() { - None - } else { - let token = self.next_token_inner()?; - // Skip whitespace here to update location - // to the beginning of the next token - self.skip_ws()?; - Some(TokenWithLocation { token, loc }) - }) - } -} - -#[derive(Clone)] -pub struct Parser<'a> { - lexer: Lexer<'a>, - syntax: Syntax, - next_token: Option, -} - -#[derive(Copy, Clone)] -enum MessageBodyParseMode { - MessageProto2, - MessageProto3, - Oneof, - ExtendProto2, - ExtendProto3, -} - -impl MessageBodyParseMode { - fn label_allowed(&self, label: Rule) -> bool { - match label { - Rule::Repeated => match *self { - MessageBodyParseMode::MessageProto2 - | MessageBodyParseMode::MessageProto3 - | MessageBodyParseMode::ExtendProto2 - | MessageBodyParseMode::ExtendProto3 => true, - MessageBodyParseMode::Oneof => false, - }, - Rule::Optional => match *self { - MessageBodyParseMode::MessageProto2 | MessageBodyParseMode::ExtendProto2 => true, - MessageBodyParseMode::MessageProto3 | MessageBodyParseMode::ExtendProto3 => true, - MessageBodyParseMode::Oneof => false, - }, - Rule::Required => match *self { - MessageBodyParseMode::MessageProto2 | MessageBodyParseMode::ExtendProto2 => true, - MessageBodyParseMode::MessageProto3 | MessageBodyParseMode::ExtendProto3 => false, - MessageBodyParseMode::Oneof => false, - }, - } - } - - fn some_label_required(&self) -> bool { - match *self { - MessageBodyParseMode::MessageProto2 | MessageBodyParseMode::ExtendProto2 => true, - MessageBodyParseMode::MessageProto3 - | MessageBodyParseMode::ExtendProto3 - | MessageBodyParseMode::Oneof => false, - } - } - - fn map_allowed(&self) -> bool { - match *self { - MessageBodyParseMode::MessageProto2 - | MessageBodyParseMode::MessageProto3 - | MessageBodyParseMode::ExtendProto2 - | MessageBodyParseMode::ExtendProto3 => true, - MessageBodyParseMode::Oneof => false, - } - } - - fn is_most_non_fields_allowed(&self) -> bool { - match *self { - MessageBodyParseMode::MessageProto2 | MessageBodyParseMode::MessageProto3 => true, - MessageBodyParseMode::ExtendProto2 - | MessageBodyParseMode::ExtendProto3 - | MessageBodyParseMode::Oneof => false, - } - } - - fn is_option_allowed(&self) -> bool { - match *self { - MessageBodyParseMode::MessageProto2 - | MessageBodyParseMode::MessageProto3 - | MessageBodyParseMode::Oneof => true, - MessageBodyParseMode::ExtendProto2 | MessageBodyParseMode::ExtendProto3 => false, - } - } -} - -#[derive(Default)] -pub struct MessageBody { - pub fields: Vec, - pub oneofs: Vec, - pub reserved_nums: Vec, - pub reserved_names: Vec, - pub messages: Vec, - pub enums: Vec, - pub options: Vec, -} - -#[derive(Copy, Clone)] -enum NumLit { - U64(u64), - F64(f64), -} - -impl NumLit { - fn to_option_value(self, sign_is_plus: bool) -> ParserResult { - Ok(match (self, sign_is_plus) { - (NumLit::U64(u), true) => ProtobufConstant::U64(u), - (NumLit::F64(f), true) => ProtobufConstant::F64(f), - (NumLit::U64(u), false) => ProtobufConstant::I64(u.neg()?), - (NumLit::F64(f), false) => ProtobufConstant::F64(-f), - }) - } -} - -impl<'a> Parser<'a> { - pub fn new(input: &'a str) -> Parser<'a> { - Parser { - lexer: Lexer { - input, - pos: 0, - loc: Loc::start(), - }, - syntax: Syntax::Proto2, - next_token: None, - } - } - - pub fn loc(&self) -> Loc { - self.next_token.clone().map_or(self.lexer.loc, |n| n.loc) - } - - fn lookahead(&mut self) -> ParserResult> { - Ok(match self.next_token { - Some(ref token) => Some(&token.token), - None => { - self.next_token = self.lexer.next_token()?; - match self.next_token { - Some(ref token) => Some(&token.token), - None => None, - } - } - }) - } - - fn lookahead_some(&mut self) -> ParserResult<&Token> { - match self.lookahead()? { - Some(token) => Ok(token), - None => Err(ParserError::UnexpectedEof), - } - } - - fn next(&mut self) -> ParserResult> { - self.lookahead()?; - Ok(self - .next_token - .take() - .map(|TokenWithLocation { token, .. }| token)) - } - - fn next_some(&mut self) -> ParserResult { - match self.next()? { - Some(token) => Ok(token), - None => Err(ParserError::UnexpectedEof), - } - } - - /// Can be called only after lookahead, otherwise it's error - fn advance(&mut self) -> ParserResult { - self.next_token - .take() - .map(|TokenWithLocation { token, .. }| token) - .ok_or(ParserError::InternalError) - } - - /// No more tokens - fn syntax_eof(&mut self) -> ParserResult { - Ok(self.lookahead()?.is_none()) - } - - fn next_token_if_map(&mut self, p: P) -> ParserResult> - where - P: FnOnce(&Token) -> Option, - { - self.lookahead()?; - let v = match self.next_token { - Some(ref token) => match p(&token.token) { - Some(v) => v, - None => return Ok(None), - }, - _ => return Ok(None), - }; - self.next_token = None; - Ok(Some(v)) - } - - fn next_token_check_map(&mut self, p: P) -> ParserResult - where - P: FnOnce(&Token) -> ParserResult, - { - self.lookahead()?; - let r = match self.next_token { - Some(ref token) => p(&token.token)?, - None => return Err(ParserError::UnexpectedEof), - }; - self.next_token = None; - Ok(r) - } - - fn next_token_if

(&mut self, p: P) -> ParserResult> - where - P: FnOnce(&Token) -> bool, - { - self.next_token_if_map(|token| if p(token) { Some(token.clone()) } else { None }) - } - - fn next_ident_if_in(&mut self, idents: &[&str]) -> ParserResult> { - let v = match self.lookahead()? { - Some(Token::Ident(next)) if idents.iter().any(|i| i == next) => next.clone(), - _ => return Ok(None), - }; - self.advance()?; - Ok(Some(v)) - } - - fn next_ident_if_eq(&mut self, word: &str) -> ParserResult { - Ok((self.next_ident_if_in(&[word])?).is_some()) - } - - pub fn next_ident_expect_eq(&mut self, word: &str) -> ParserResult<()> { - if self.next_ident_if_eq(word)? { - Ok(()) - } else { - Err(ParserError::ExpectNamedIdent(word.to_owned())) - } - } - - fn next_ident_if_eq_error(&mut self, word: &str) -> ParserResult<()> { - if self.clone().next_ident_if_eq(word)? { - return Err(ParserError::IncorrectInput); - } - Ok(()) - } - - fn next_symbol_if_eq(&mut self, symbol: char) -> ParserResult { - Ok( - (self.next_token_if(|token| matches!(*token, Token::Symbol(c) if c == symbol))?) - .is_some(), - ) - } - - fn next_symbol_expect_eq(&mut self, symbol: char) -> ParserResult<()> { - if self.lookahead_is_symbol(symbol)? { - self.advance()?; - Ok(()) - } else { - Err(ParserError::ExpectChar(symbol)) - } - } - - fn lookahead_if_symbol(&mut self) -> ParserResult> { - Ok(match self.lookahead()? { - Some(&Token::Symbol(c)) => Some(c), - _ => None, - }) - } - - fn lookahead_is_symbol(&mut self, symbol: char) -> ParserResult { - Ok(self.lookahead_if_symbol()? == Some(symbol)) - } - - // Protobuf grammar - - fn next_ident(&mut self) -> ParserResult { - self.next_token_check_map(|token| match *token { - Token::Ident(ref ident) => Ok(ident.clone()), - _ => Err(ParserError::ExpectIdent), - }) - } - - fn next_str_lit(&mut self) -> ParserResult { - self.next_token_check_map(|token| match *token { - Token::StrLit(ref str_lit) => Ok(str_lit.clone()), - _ => Err(ParserError::IncorrectInput), - }) - } - - // fullIdent = ident { "." ident } - fn next_full_ident(&mut self) -> ParserResult { - let mut full_ident = String::new(); - // https://github.com/google/protobuf/issues/4563 - if self.next_symbol_if_eq('.')? { - full_ident.push('.'); - } - full_ident.push_str(&self.next_ident()?); - while self.next_symbol_if_eq('.')? { - full_ident.push('.'); - full_ident.push_str(&self.next_ident()?); - } - Ok(full_ident) - } - - // emptyStatement = ";" - fn next_empty_statement_opt(&mut self) -> ParserResult> { - if self.next_symbol_if_eq(';')? { - Ok(Some(())) - } else { - Ok(None) - } - } - - // messageName = ident - // enumName = ident - // messageType = [ "." ] { ident "." } messageName - // enumType = [ "." ] { ident "." } enumName - fn next_message_or_enum_type(&mut self) -> ParserResult { - let mut full_name = String::new(); - if self.next_symbol_if_eq('.')? { - full_name.push('.'); - } - full_name.push_str(&self.next_ident()?); - while self.next_symbol_if_eq('.')? { - full_name.push('.'); - full_name.push_str(&self.next_ident()?); - } - Ok(full_name) - } - - fn is_ascii_uppercase(c: char) -> bool { - c.is_ascii_uppercase() - } - - // groupName = capitalLetter { letter | decimalDigit | "_" } - fn next_group_name(&mut self) -> ParserResult { - // lexer cannot distinguish between group name and other ident - let mut clone = self.clone(); - let ident = clone.next_ident()?; - if !Parser::is_ascii_uppercase(ident.chars().next().unwrap()) { - return Err(ParserError::GroupNameShouldStartWithUpperCase); - } - *self = clone; - Ok(ident) - } - - // Boolean - - // boolLit = "true" | "false" - fn next_bool_lit_opt(&mut self) -> ParserResult> { - Ok(if self.next_ident_if_eq("true")? { - Some(true) - } else if self.next_ident_if_eq("false")? { - Some(false) - } else { - None - }) - } - - // Constant - - fn next_num_lit(&mut self) -> ParserResult { - self.next_token_check_map(|token| token.to_num_lit()) - } - - // constant = fullIdent | ( [ "-" | "+" ] intLit ) | ( [ "-" | "+" ] floatLit ) | - // strLit | boolLit - fn next_constant(&mut self) -> ParserResult { - // https://github.com/google/protobuf/blob/a21f225824e994ebd35e8447382ea4e0cd165b3c/src/google/protobuf/unittest_custom_options.proto#L350 - if self.lookahead_is_symbol('{')? { - return Ok(ProtobufConstant::BracedExpr(self.next_braces()?)); - } - - if let Some(b) = self.next_bool_lit_opt()? { - return Ok(ProtobufConstant::Bool(b)); - } - - if let Token::Symbol(c) = *(self.lookahead_some()?) { - if c == '+' || c == '-' { - self.advance()?; - let sign = c == '+'; - return self.next_num_lit()?.to_option_value(sign); - } - } - - if let Some(r) = self.next_token_if_map(|token| match *token { - Token::StrLit(ref s) => Some(ProtobufConstant::String(s.clone())), - _ => None, - })? { - return Ok(r); - } - - match self.lookahead_some()? { - &Token::IntLit(..) | &Token::FloatLit(..) => { - return self.next_num_lit()?.to_option_value(true); - } - &Token::Ident(..) => { - return Ok(ProtobufConstant::Ident(self.next_full_ident()?)); - } - _ => {} - } - - Err(ParserError::ExpectConstant) - } - - fn next_int_lit(&mut self) -> ParserResult { - self.next_token_check_map(|token| match *token { - Token::IntLit(i) => Ok(i), - _ => Err(ParserError::IncorrectInput), - }) - } - - // Syntax - - // syntax = "syntax" "=" quote "proto2" quote ";" - // syntax = "syntax" "=" quote "proto3" quote ";" - fn next_syntax(&mut self) -> ParserResult> { - if self.next_ident_if_eq("syntax")? { - self.next_symbol_expect_eq('=')?; - let syntax_str = self.next_str_lit()?.decode_utf8()?; - let syntax = if syntax_str == "proto2" { - Syntax::Proto2 - } else if syntax_str == "proto3" { - Syntax::Proto3 - } else { - return Err(ParserError::UnknownSyntax); - }; - self.next_symbol_expect_eq(';')?; - Ok(Some(syntax)) - } else { - Ok(None) - } - } - - // Import Statement - - // import = "import" [ "weak" | "public" ] strLit ";" - fn next_import_opt(&mut self) -> ParserResult> { - if self.next_ident_if_eq("import")? { - self.next_ident_if_in(&["weak", "public"])?; - let import_path = self.next_str_lit()?.decode_utf8()?; - self.next_symbol_expect_eq(';')?; - Ok(Some(import_path)) - } else { - Ok(None) - } - } - - // Package - - // package = "package" fullIdent ";" - fn next_package_opt(&mut self) -> ParserResult> { - if self.next_ident_if_eq("package")? { - let package = self.next_full_ident()?; - self.next_symbol_expect_eq(';')?; - Ok(Some(package)) - } else { - Ok(None) - } - } - - // Option - - fn next_ident_or_braced(&mut self) -> ParserResult { - let mut ident_or_braced = String::new(); - if self.next_symbol_if_eq('(')? { - ident_or_braced.push('('); - ident_or_braced.push_str(&self.next_full_ident()?); - self.next_symbol_expect_eq(')')?; - ident_or_braced.push(')'); - } else { - ident_or_braced.push_str(&self.next_ident()?); - } - Ok(ident_or_braced) - } - - // https://github.com/google/protobuf/issues/4563 - // optionName = ( ident | "(" fullIdent ")" ) { "." ident } - fn next_option_name(&mut self) -> ParserResult { - let mut option_name = String::new(); - option_name.push_str(&self.next_ident_or_braced()?); - while self.next_symbol_if_eq('.')? { - option_name.push('.'); - option_name.push_str(&self.next_ident_or_braced()?); - } - Ok(option_name) - } - - // option = "option" optionName "=" constant ";" - fn next_option_opt(&mut self) -> ParserResult> { - if self.next_ident_if_eq("option")? { - let name = self.next_option_name()?; - self.next_symbol_expect_eq('=')?; - let value = self.next_constant()?; - self.next_symbol_expect_eq(';')?; - Ok(Some(ProtobufOption { name, value })) - } else { - Ok(None) - } - } - - // Fields - - // label = "required" | "optional" | "repeated" - fn next_label(&mut self, mode: MessageBodyParseMode) -> ParserResult { - let map = &[ - ("optional", Rule::Optional), - ("required", Rule::Required), - ("repeated", Rule::Repeated), - ]; - for &(name, value) in map { - let mut clone = self.clone(); - if clone.next_ident_if_eq(name)? { - if !mode.label_allowed(value) { - return Err(ParserError::LabelNotAllowed); - } - - *self = clone; - return Ok(value); - } - } - - if mode.some_label_required() { - Err(ParserError::LabelRequired) - } else { - Ok(Rule::Optional) - } - } - - fn next_field_type(&mut self) -> ParserResult { - let simple = &[ - ("int32", FieldType::Int32), - ("int64", FieldType::Int64), - ("uint32", FieldType::Uint32), - ("uint64", FieldType::Uint64), - ("sint32", FieldType::Sint32), - ("sint64", FieldType::Sint64), - ("fixed32", FieldType::Fixed32), - ("sfixed32", FieldType::Sfixed32), - ("fixed64", FieldType::Fixed64), - ("sfixed64", FieldType::Sfixed64), - ("bool", FieldType::Bool), - ("string", FieldType::String), - ("bytes", FieldType::Bytes), - ("float", FieldType::Float), - ("double", FieldType::Double), - ]; - for &(n, ref t) in simple { - if self.next_ident_if_eq(n)? { - return Ok(t.clone()); - } - } - - if let Some(t) = self.next_map_field_type_opt()? { - return Ok(t); - } - - let message_or_enum = self.next_message_or_enum_type()?; - Ok(FieldType::MessageOrEnum(message_or_enum)) - } - - fn next_field_number(&mut self) -> ParserResult { - self.next_token_check_map(|token| match *token { - Token::IntLit(i) => i.to_i32(), - _ => Err(ParserError::IncorrectInput), - }) - } - - // fieldOption = optionName "=" constant - fn next_field_option(&mut self) -> ParserResult { - let name = self.next_option_name()?; - self.next_symbol_expect_eq('=')?; - let value = self.next_constant()?; - Ok(ProtobufOption { name, value }) - } - - // fieldOptions = fieldOption { "," fieldOption } - fn next_field_options(&mut self) -> ParserResult> { - let mut options = vec![self.next_field_option()?]; - - while self.next_symbol_if_eq(',')? { - options.push(self.next_field_option()?); - } - - Ok(options) - } - - // field = label type fieldName "=" fieldNumber [ "[" fieldOptions "]" ] ";" - // group = label "group" groupName "=" fieldNumber messageBody - fn next_field(&mut self, mode: MessageBodyParseMode) -> ParserResult { - let rule = if self.clone().next_ident_if_eq("map")? { - if !mode.map_allowed() { - return Err(ParserError::MapFieldNotAllowed); - } - Rule::Optional - } else { - self.next_label(mode)? - }; - if self.next_ident_if_eq("group")? { - let name = self.next_group_name()?; - self.next_symbol_expect_eq('=')?; - let number = self.next_field_number()?; - - let mode = match self.syntax { - Syntax::Proto2 => MessageBodyParseMode::MessageProto2, - Syntax::Proto3 => MessageBodyParseMode::MessageProto3, - }; - - let MessageBody { fields, .. } = self.next_message_body(mode)?; - - Ok(Field { - name, - rule, - typ: FieldType::Group(fields), - number, - options: Vec::new(), - }) - } else { - let typ = self.next_field_type()?; - let name = self.next_ident()?; - self.next_symbol_expect_eq('=')?; - let number = self.next_field_number()?; - - let mut options = Vec::new(); - - if self.next_symbol_if_eq('[')? { - for o in self.next_field_options()? { - options.push(o); - } - self.next_symbol_expect_eq(']')?; - } - self.next_symbol_expect_eq(';')?; - Ok(Field { - name, - rule, - typ, - number, - options, - }) - } - } - - // oneof = "oneof" oneofName "{" { oneofField | emptyStatement } "}" - // oneofField = type fieldName "=" fieldNumber [ "[" fieldOptions "]" ] ";" - fn next_oneof_opt(&mut self) -> ParserResult> { - if self.next_ident_if_eq("oneof")? { - let name = self.next_ident()?; - let MessageBody { fields, .. } = self.next_message_body(MessageBodyParseMode::Oneof)?; - Ok(Some(OneOf { name, fields })) - } else { - Ok(None) - } - } - - // mapField = "map" "<" keyType "," type ">" mapName "=" fieldNumber [ "[" fieldOptions "]" ] ";" - // keyType = "int32" | "int64" | "uint32" | "uint64" | "sint32" | "sint64" | - // "fixed32" | "fixed64" | "sfixed32" | "sfixed64" | "bool" | "string" - fn next_map_field_type_opt(&mut self) -> ParserResult> { - if self.next_ident_if_eq("map")? { - self.next_symbol_expect_eq('<')?; - // TODO: restrict key types - let key = self.next_field_type()?; - self.next_symbol_expect_eq(',')?; - let value = self.next_field_type()?; - self.next_symbol_expect_eq('>')?; - Ok(Some(FieldType::Map(Box::new((key, value))))) - } else { - Ok(None) - } - } - - // Extensions and Reserved - - // Extensions - - // range = intLit [ "to" ( intLit | "max" ) ] - fn next_range(&mut self) -> ParserResult { - let from = self.next_field_number()?; - let to = if self.next_ident_if_eq("to")? { - if self.next_ident_if_eq("max")? { - i32::MAX - } else { - self.next_field_number()? - } - } else { - from - }; - Ok(FieldNumberRange { from, to }) - } - - // ranges = range { "," range } - fn next_ranges(&mut self) -> ParserResult> { - let mut ranges = vec![self.next_range()?]; - while self.next_symbol_if_eq(',')? { - ranges.push(self.next_range()?); - } - Ok(ranges) - } - - // extensions = "extensions" ranges ";" - fn next_extensions_opt(&mut self) -> ParserResult>> { - if self.next_ident_if_eq("extensions")? { - Ok(Some(self.next_ranges()?)) - } else { - Ok(None) - } - } - - // Reserved - - // Grammar is incorrect: https://github.com/google/protobuf/issues/4558 - // reserved = "reserved" ( ranges | fieldNames ) ";" - // fieldNames = fieldName { "," fieldName } - fn next_reserved_opt(&mut self) -> ParserResult, Vec)>> { - if self.next_ident_if_eq("reserved")? { - let (ranges, names) = if let Token::StrLit(..) = *(self.lookahead_some()?) { - let mut names = vec![self.next_str_lit()?.decode_utf8()?]; - while self.next_symbol_if_eq(',')? { - names.push(self.next_str_lit()?.decode_utf8()?); - } - (Vec::new(), names) - } else { - (self.next_ranges()?, Vec::new()) - }; - - self.next_symbol_expect_eq(';')?; - - Ok(Some((ranges, names))) - } else { - Ok(None) - } - } - - // Top Level definitions - - // Enum definition - - // enumValueOption = optionName "=" constant - fn next_enum_value_option(&mut self) -> ParserResult<()> { - self.next_option_name()?; - self.next_symbol_expect_eq('=')?; - self.next_constant()?; - Ok(()) - } - - // https://github.com/google/protobuf/issues/4561 - fn next_enum_value(&mut self) -> ParserResult { - let minus = self.next_symbol_if_eq('-')?; - let lit = self.next_int_lit()?; - Ok(if minus { - let unsigned = lit.to_i64()?; - match unsigned.checked_neg() { - Some(neg) => neg.to_i32()?, - None => return Err(ParserError::IntegerOverflow), - } - } else { - lit.to_i32()? - }) - } - - // enumField = ident "=" intLit [ "[" enumValueOption { "," enumValueOption } "]" ]";" - fn next_enum_field(&mut self) -> ParserResult { - let name = self.next_ident()?; - self.next_symbol_expect_eq('=')?; - let number = self.next_enum_value()?; - if self.next_symbol_if_eq('[')? { - self.next_enum_value_option()?; - while self.next_symbol_if_eq(',')? { - self.next_enum_value_option()?; - } - self.next_symbol_expect_eq(']')?; - } - - Ok(EnumValue { name, number }) - } - - // enum = "enum" enumName enumBody - // enumBody = "{" { option | enumField | emptyStatement } "}" - fn next_enum_opt(&mut self) -> ParserResult> { - if self.next_ident_if_eq("enum")? { - let name = self.next_ident()?; - - let mut values = Vec::new(); - let mut options = Vec::new(); - - self.next_symbol_expect_eq('{')?; - while self.lookahead_if_symbol()? != Some('}') { - // emptyStatement - if self.next_symbol_if_eq(';')? { - continue; - } - - if let Some(o) = self.next_option_opt()? { - options.push(o); - continue; - } - - values.push(self.next_enum_field()?); - } - self.next_symbol_expect_eq('}')?; - Ok(Some(Enumeration { - name, - values, - options, - })) - } else { - Ok(None) - } - } - - // Message definition - - // messageBody = "{" { field | enum | message | extend | extensions | group | - // option | oneof | mapField | reserved | emptyStatement } "}" - fn next_message_body(&mut self, mode: MessageBodyParseMode) -> ParserResult { - self.next_symbol_expect_eq('{')?; - - let mut r = MessageBody::default(); - - while self.lookahead_if_symbol()? != Some('}') { - // emptyStatement - if self.next_symbol_if_eq(';')? { - continue; - } - - if mode.is_most_non_fields_allowed() { - if let Some((field_nums, field_names)) = self.next_reserved_opt()? { - r.reserved_nums.extend(field_nums); - r.reserved_names.extend(field_names); - continue; - } - - if let Some(oneof) = self.next_oneof_opt()? { - r.oneofs.push(oneof); - continue; - } - - if let Some(_extensions) = self.next_extensions_opt()? { - continue; - } - - if let Some(_extend) = self.next_extend_opt()? { - continue; - } - - if let Some(nested_message) = self.next_message_opt()? { - r.messages.push(nested_message); - continue; - } - - if let Some(nested_enum) = self.next_enum_opt()? { - r.enums.push(nested_enum); - continue; - } - } else { - self.next_ident_if_eq_error("reserved")?; - self.next_ident_if_eq_error("oneof")?; - self.next_ident_if_eq_error("extensions")?; - self.next_ident_if_eq_error("extend")?; - self.next_ident_if_eq_error("message")?; - self.next_ident_if_eq_error("enum")?; - } - - if mode.is_option_allowed() { - if let Some(option) = self.next_option_opt()? { - r.options.push(option); - continue; - } - } else { - self.next_ident_if_eq_error("option")?; - } - - r.fields.push(self.next_field(mode)?); - } - - self.next_symbol_expect_eq('}')?; - - Ok(r) - } - - // message = "message" messageName messageBody - fn next_message_opt(&mut self) -> ParserResult> { - if self.next_ident_if_eq("message")? { - let name = self.next_ident()?; - - let mode = match self.syntax { - Syntax::Proto2 => MessageBodyParseMode::MessageProto2, - Syntax::Proto3 => MessageBodyParseMode::MessageProto3, - }; - - let MessageBody { - fields, - oneofs, - reserved_nums, - reserved_names, - messages, - enums, - options, - } = self.next_message_body(mode)?; - - Ok(Some(Message { - name, - fields, - oneofs, - reserved_nums, - reserved_names, - messages, - enums, - options, - })) - } else { - Ok(None) - } - } - - // Extend - - // extend = "extend" messageType "{" {field | group | emptyStatement} "}" - fn next_extend_opt(&mut self) -> ParserResult>> { - let mut clone = self.clone(); - if clone.next_ident_if_eq("extend")? { - // According to spec `extend` is only for `proto2`, but it is used in `proto3` - // https://github.com/google/protobuf/issues/4610 - - *self = clone; - - let extendee = self.next_message_or_enum_type()?; - - let mode = match self.syntax { - Syntax::Proto2 => MessageBodyParseMode::ExtendProto2, - Syntax::Proto3 => MessageBodyParseMode::ExtendProto3, - }; - - let MessageBody { fields, .. } = self.next_message_body(mode)?; - - let extensions = fields - .into_iter() - .map(|field| { - let extendee = extendee.clone(); - Extension { extendee, field } - }) - .collect(); - - Ok(Some(extensions)) - } else { - Ok(None) - } - } - - // Service definition - - fn next_braces(&mut self) -> ParserResult { - let mut r = String::new(); - self.next_symbol_expect_eq('{')?; - r.push('{'); - loop { - if self.lookahead_if_symbol()? == Some('{') { - r.push_str(&self.next_braces()?); - continue; - } - let next = self.next_some()?; - r.push_str(&next.format()); - if let Token::Symbol('}') = next { - break; - } - } - Ok(r) - } - - fn next_options_or_colon(&mut self) -> ParserResult> { - let mut options = Vec::new(); - if self.next_symbol_if_eq('{')? { - while self.lookahead_if_symbol()? != Some('}') { - if let Some(option) = self.next_option_opt()? { - options.push(option); - continue; - } - - if let Some(()) = self.next_empty_statement_opt()? { - continue; - } - - return Err(ParserError::IncorrectInput); - } - self.next_symbol_expect_eq('}')?; - } else { - self.next_symbol_expect_eq(';')?; - } - - Ok(options) - } - - // stream = "stream" streamName "(" messageType "," messageType ")" - // (( "{" { option | emptyStatement } "}") | ";" ) - fn next_stream_opt(&mut self) -> ParserResult> { - assert_eq!(Syntax::Proto2, self.syntax); - if self.next_ident_if_eq("stream")? { - let name = self.next_ident()?; - self.next_symbol_expect_eq('(')?; - let input_type = self.next_ident()?; - self.next_symbol_expect_eq(',')?; - let output_type = self.next_ident()?; - self.next_symbol_expect_eq(')')?; - let options = self.next_options_or_colon()?; - Ok(Some(Method { - name, - input_type, - output_type, - client_streaming: true, - server_streaming: true, - options, - })) - } else { - Ok(None) - } - } - - // rpc = "rpc" rpcName "(" [ "stream" ] messageType ")" - // "returns" "(" [ "stream" ] messageType ")" - // (( "{" { option | emptyStatement } "}" ) | ";" ) - fn next_rpc_opt(&mut self) -> ParserResult> { - if self.next_ident_if_eq("rpc")? { - let name = self.next_ident()?; - self.next_symbol_expect_eq('(')?; - let client_streaming = self.next_ident_if_eq("stream")?; - let input_type = self.next_message_or_enum_type()?; - self.next_symbol_expect_eq(')')?; - self.next_ident_expect_eq("returns")?; - self.next_symbol_expect_eq('(')?; - let server_streaming = self.next_ident_if_eq("stream")?; - let output_type = self.next_message_or_enum_type()?; - self.next_symbol_expect_eq(')')?; - let options = self.next_options_or_colon()?; - Ok(Some(Method { - name, - input_type, - output_type, - client_streaming, - server_streaming, - options, - })) - } else { - Ok(None) - } - } - - // proto2: - // service = "service" serviceName "{" { option | rpc | stream | emptyStatement } "}" - // - // proto3: - // service = "service" serviceName "{" { option | rpc | emptyStatement } "}" - fn next_service_opt(&mut self) -> ParserResult> { - if self.next_ident_if_eq("service")? { - let name = self.next_ident()?; - let mut methods = Vec::new(); - let mut options = Vec::new(); - self.next_symbol_expect_eq('{')?; - while self.lookahead_if_symbol()? != Some('}') { - if let Some(method) = self.next_rpc_opt()? { - methods.push(method); - continue; - } - - if self.syntax == Syntax::Proto2 { - if let Some(method) = self.next_stream_opt()? { - methods.push(method); - continue; - } - } - - if let Some(o) = self.next_option_opt()? { - options.push(o); - continue; - } - - if let Some(()) = self.next_empty_statement_opt()? { - continue; - } - - return Err(ParserError::IncorrectInput); - } - self.next_symbol_expect_eq('}')?; - Ok(Some(Service { - name, - methods, - options, - })) - } else { - Ok(None) - } - } - - // Proto file - - // proto = syntax { import | package | option | topLevelDef | emptyStatement } - // topLevelDef = message | enum | extend | service - pub fn next_proto(&mut self) -> ParserResult { - let syntax = self.next_syntax()?.unwrap_or(Syntax::Proto2); - self.syntax = syntax; - - let mut import_paths = Vec::new(); - let mut package = String::new(); - let mut messages = Vec::new(); - let mut enums = Vec::new(); - let mut extensions = Vec::new(); - let mut options = Vec::new(); - let mut services = Vec::new(); - - while !self.syntax_eof()? { - if let Some(import_path) = self.next_import_opt()? { - import_paths.push(import_path); - continue; - } - - if let Some(next_package) = self.next_package_opt()? { - package = next_package.to_owned(); - continue; - } - - if let Some(option) = self.next_option_opt()? { - options.push(option); - continue; - } - - if let Some(message) = self.next_message_opt()? { - messages.push(message); - continue; - } - - if let Some(enumeration) = self.next_enum_opt()? { - enums.push(enumeration); - continue; - } - - if let Some(more_extensions) = self.next_extend_opt()? { - extensions.extend(more_extensions); - continue; - } - - if let Some(service) = self.next_service_opt()? { - services.push(service); - continue; - } - - if self.next_symbol_if_eq(';')? { - continue; - } - - return Err(ParserError::IncorrectInput); - } - - Ok(FileDescriptor { - import_paths, - package, - syntax, - messages, - enums, - extensions, - services, - options, - }) - } -} - -#[cfg(test)] -mod test { - use super::*; - - fn lex(input: &str, parse_what: P) -> R - where - P: FnOnce(&mut Lexer) -> ParserResult, - { - let mut lexer = Lexer { - input, - pos: 0, - loc: Loc::start(), - }; - let r = parse_what(&mut lexer).unwrap_or_else(|_| panic!("lexer failed at {}", lexer.loc)); - assert!(lexer.eof(), "check eof failed at {}", lexer.loc); - r - } - - fn lex_opt(input: &str, parse_what: P) -> R - where - P: FnOnce(&mut Lexer) -> ParserResult>, - { - let mut lexer = Lexer { - input, - pos: 0, - loc: Loc::start(), - }; - let o = parse_what(&mut lexer).unwrap_or_else(|_| panic!("lexer failed at {}", lexer.loc)); - let r = o.unwrap_or_else(|| panic!("lexer returned none at {}", lexer.loc)); - assert!(lexer.eof(), "check eof failed at {}", lexer.loc); - r - } - - fn parse(input: &str, parse_what: P) -> R - where - P: FnOnce(&mut Parser) -> ParserResult, - { - let mut parser = Parser::new(input); - let r = - parse_what(&mut parser).unwrap_or_else(|_| panic!("parse failed at {}", parser.loc())); - let eof = parser - .syntax_eof() - .unwrap_or_else(|_| panic!("check eof failed at {}", parser.loc())); - assert!(eof, "{}", parser.loc()); - r - } - - fn parse_opt(input: &str, parse_what: P) -> R - where - P: FnOnce(&mut Parser) -> ParserResult>, - { - let mut parser = Parser::new(input); - let o = - parse_what(&mut parser).unwrap_or_else(|_| panic!("parse failed at {}", parser.loc())); - let r = o.unwrap_or_else(|| panic!("parser returned none at {}", parser.loc())); - assert!(parser.syntax_eof().unwrap()); - r - } - - #[test] - fn test_lexer_int_lit() { - let msg = r#"10"#; - let mess = lex_opt(msg, |p| p.next_int_lit_opt()); - assert_eq!(10, mess); - } - - #[test] - fn test_lexer_float_lit() { - let msg = r#"12.3"#; - let mess = lex(msg, |p| p.next_token_inner()); - assert_eq!(Token::FloatLit(12.3), mess); - } - - #[test] - fn test_ident() { - let msg = r#" aabb_c "#; - let mess = parse(msg, |p| p.next_ident()); - assert_eq!("aabb_c", mess); - } - - #[test] - fn test_str_lit() { - let msg = r#" "a\nb" "#; - let mess = parse(msg, |p| p.next_str_lit()); - assert_eq!( - StrLit { - escaped: r#"a\nb"#.to_owned() - }, - mess - ); - } - - #[test] - fn test_syntax() { - let msg = r#" syntax = "proto3"; "#; - let mess = parse_opt(msg, |p| p.next_syntax()); - assert_eq!(Syntax::Proto3, mess); - } - - #[test] - fn test_field_default_value_int() { - let msg = r#" optional int64 f = 4 [default = 12]; "#; - let mess = parse(msg, |p| p.next_field(MessageBodyParseMode::MessageProto2)); - assert_eq!("f", mess.name); - assert_eq!("default", mess.options[0].name); - assert_eq!("12", mess.options[0].value.format()); - } - - #[test] - fn test_field_default_value_float() { - let msg = r#" optional float f = 2 [default = 10.0]; "#; - let mess = parse(msg, |p| p.next_field(MessageBodyParseMode::MessageProto2)); - assert_eq!("f", mess.name); - assert_eq!("default", mess.options[0].name); - assert_eq!("10", mess.options[0].value.format()); - } - - #[test] - fn test_message() { - let msg = r#"message ReferenceData - { - repeated ScenarioInfo scenarioSet = 1; - repeated CalculatedObjectInfo calculatedObjectSet = 2; - repeated RiskFactorList riskFactorListSet = 3; - repeated RiskMaturityInfo riskMaturitySet = 4; - repeated IndicatorInfo indicatorSet = 5; - repeated RiskStrikeInfo riskStrikeSet = 6; - repeated FreeProjectionList freeProjectionListSet = 7; - repeated ValidationProperty ValidationSet = 8; - repeated CalcProperties calcPropertiesSet = 9; - repeated MaturityInfo maturitySet = 10; - }"#; - - let mess = parse_opt(msg, |p| p.next_message_opt()); - assert_eq!(10, mess.fields.len()); - } - - #[test] - fn test_enum() { - let msg = r#"enum PairingStatus { - DEALPAIRED = 0; - INVENTORYORPHAN = 1; - CALCULATEDORPHAN = 2; - CANCELED = 3; - }"#; - - let enumeration = parse_opt(msg, |p| p.next_enum_opt()); - assert_eq!(4, enumeration.values.len()); - } - - #[test] - fn test_ignore() { - let msg = r#"option optimize_for = SPEED;"#; - - parse_opt(msg, |p| p.next_option_opt()); - } - - #[test] - fn test_import() { - let msg = r#"syntax = "proto3"; - import "test_import_nested_imported_pb.proto"; - message ContainsImportedNested { - ContainerForNested.NestedMessage m = 1; - ContainerForNested.NestedEnum e = 2; - } - "#; - let desc = parse(msg, |p| p.next_proto()); - - assert_eq!( - vec!["test_import_nested_imported_pb.proto"], - desc.import_paths - ); - } - - #[test] - fn test_package() { - let msg = r#" - package foo.bar; - message ContainsImportedNested { - optional ContainerForNested.NestedMessage m = 1; - optional ContainerForNested.NestedEnum e = 2; - } - "#; - let desc = parse(msg, |p| p.next_proto()); - assert_eq!("foo.bar".to_string(), desc.package); - } - - #[test] - fn test_nested_message() { - let msg = r#"message A - { - message B { - repeated int32 a = 1; - optional string b = 2; - } - optional string b = 1; - }"#; - - let mess = parse_opt(msg, |p| p.next_message_opt()); - assert_eq!(1, mess.messages.len()); - } - - #[test] - fn test_map() { - let msg = r#"message A - { - optional map b = 1; - }"#; - - let mess = parse_opt(msg, |p| p.next_message_opt()); - assert_eq!(1, mess.fields.len()); - match mess.fields[0].typ { - FieldType::Map(ref f) => match &**f { - &(FieldType::String, FieldType::Int32) => (), - ref f => panic!("Expecting Map found {:?}", f), - }, - ref f => panic!("Expecting map, got {:?}", f), - } - } - - #[test] - fn test_oneof() { - let msg = r#"message A - { - optional int32 a1 = 1; - oneof a_oneof { - string a2 = 2; - int32 a3 = 3; - bytes a4 = 4; - } - repeated bool a5 = 5; - }"#; - - let mess = parse_opt(msg, |p| p.next_message_opt()); - assert_eq!(1, mess.oneofs.len()); - assert_eq!(3, mess.oneofs[0].fields.len()); - } - - #[test] - fn test_reserved() { - let msg = r#"message Sample { - reserved 4, 15, 17 to 20, 30; - reserved "foo", "bar"; - optional uint64 age =1; - required bytes name =2; - }"#; - - let mess = parse_opt(msg, |p| p.next_message_opt()); - assert_eq!( - vec![ - FieldNumberRange { from: 4, to: 4 }, - FieldNumberRange { from: 15, to: 15 }, - FieldNumberRange { from: 17, to: 20 }, - FieldNumberRange { from: 30, to: 30 } - ], - mess.reserved_nums - ); - assert_eq!( - vec!["foo".to_string(), "bar".to_string()], - mess.reserved_names - ); - assert_eq!(2, mess.fields.len()); - } - - #[test] - fn test_default_value_int() { - let msg = r#"message Sample { - optional int32 x = 1 [default = 17]; - }"#; - - let mess = parse_opt(msg, |p| p.next_message_opt()); - assert_eq!("default", mess.fields[0].options[0].name); - assert_eq!("17", mess.fields[0].options[0].value.format()); - } - - #[test] - fn test_default_value_string() { - let msg = r#"message Sample { - optional string x = 1 [default = "ab\nc d\"g\'h\0\"z"]; - }"#; - - let mess = parse_opt(msg, |p| p.next_message_opt()); - assert_eq!( - r#""ab\nc d\"g\'h\0\"z""#, - mess.fields[0].options[0].value.format() - ); - } - - #[test] - fn test_default_value_bytes() { - let msg = r#"message Sample { - optional bytes x = 1 [default = "ab\nc d\xfeE\"g\'h\0\"z"]; - }"#; - - let mess = parse_opt(msg, |p| p.next_message_opt()); - assert_eq!( - r#""ab\nc d\xfeE\"g\'h\0\"z""#, - mess.fields[0].options[0].value.format() - ); - } - - #[test] - fn test_group() { - let msg = r#"message MessageWithGroup { - optional string aaa = 1; - repeated group Identifier = 18 { - optional int32 iii = 19; - optional string sss = 20; - } - required int bbb = 3; - }"#; - let mess = parse_opt(msg, |p| p.next_message_opt()); - - assert_eq!("Identifier", mess.fields[1].name); - if let FieldType::Group(ref group_fields) = mess.fields[1].typ { - assert_eq!(2, group_fields.len()); - } else { - panic!("expecting group"); - } - - assert_eq!("bbb", mess.fields[2].name); - } - - #[test] - fn test_incorrect_file_descriptor() { - let msg = r#" - message Foo {} - dfgdg - "#; - - let err = FileDescriptor::parse(msg).expect_err("err"); - assert_eq!(3, err.line); - } - - #[test] - fn test_extend() { - let proto = r#" - syntax = "proto2"; - extend google.protobuf.FileOptions { - optional bool foo = 17001; - optional string bar = 17002; - } - extend google.protobuf.MessageOptions { - optional bool baz = 17003; - } - "#; - - let fd = FileDescriptor::parse(proto).expect("fd"); - assert_eq!(3, fd.extensions.len()); - assert_eq!("google.protobuf.FileOptions", fd.extensions[0].extendee); - assert_eq!("google.protobuf.FileOptions", fd.extensions[1].extendee); - assert_eq!("google.protobuf.MessageOptions", fd.extensions[2].extendee); - assert_eq!(17003, fd.extensions[2].field.number); - } -} diff --git a/ttrpc-codegen/src/str_lit.rs b/ttrpc-codegen/src/str_lit.rs deleted file mode 100644 index b355b443..00000000 --- a/ttrpc-codegen/src/str_lit.rs +++ /dev/null @@ -1,40 +0,0 @@ -use crate::parser::{Lexer, Loc, ParserError}; - -#[derive(Debug)] -pub enum StrLitDecodeError { - Error, -} - -impl From for StrLitDecodeError { - fn from(_: ParserError) -> Self { - StrLitDecodeError::Error - } -} - -pub type StrLitDecodeResult = Result; - -/// String literal, both `string` and `bytes`. -#[derive(Clone, Eq, PartialEq, Debug)] -pub struct StrLit { - pub escaped: String, -} - -impl StrLit { - /// May fail if not valid UTF8 - pub fn decode_utf8(&self) -> StrLitDecodeResult { - let mut lexer = Lexer { - input: &self.escaped, - pos: 0, - loc: Loc::start(), - }; - let mut r = String::new(); - while !lexer.eof() { - r.push(lexer.next_char_value()?); - } - Ok(r) - } - - pub fn quoted(&self) -> String { - format!("\"{}\"", self.escaped) - } -} diff --git a/ttrpc-codegen/tests/protos/well_known.proto b/ttrpc-codegen/tests/protos/well_known.proto new file mode 100644 index 00000000..24484d09 --- /dev/null +++ b/ttrpc-codegen/tests/protos/well_known.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +package well_known; + +import "google/protobuf/descriptor.proto"; +import "google/protobuf/timestamp.proto"; + +message TimestampedRequest { + google.protobuf.Timestamp created_at = 1; +} + +service Clock { + rpc Now(google.protobuf.Timestamp) returns (google.protobuf.Timestamp); + rpc Describe(google.protobuf.FileDescriptorProto) returns (google.protobuf.FileDescriptorProto); + rpc ExtensionRange(google.protobuf.DescriptorProto.ExtensionRange) returns (google.protobuf.DescriptorProto.ExtensionRange); + rpc Echo(TimestampedRequest) returns (TimestampedRequest); +} diff --git a/ttrpc-codegen/tests/well_known.rs b/ttrpc-codegen/tests/well_known.rs new file mode 100644 index 00000000..8b37eeec --- /dev/null +++ b/ttrpc-codegen/tests/well_known.rs @@ -0,0 +1,87 @@ +use std::collections::HashSet; +use std::fs; +use std::path::{Path, PathBuf}; + +use tempfile::TempDir; +use ttrpc_codegen::{parse_and_typecheck, Codegen, Customize}; + +fn proto_dir() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/protos") +} + +fn proto_input() -> PathBuf { + proto_dir().join("well_known.proto") +} + +#[test] +fn parses_embedded_protobuf_types() { + let include = proto_dir(); + let input = proto_input(); + let parsed = parse_and_typecheck(&[include.as_path()], &[input.as_path()]).unwrap(); + + assert_eq!(vec!["well_known.proto"], parsed.relative_paths); + let descriptor_names: HashSet<_> = parsed + .file_descriptors + .iter() + .map(|descriptor| descriptor.name()) + .collect(); + assert_eq!( + HashSet::from([ + "well_known.proto", + "google/protobuf/descriptor.proto", + "google/protobuf/timestamp.proto", + ]), + descriptor_names + ); +} + +#[test] +fn generates_runtime_paths_for_embedded_types() { + for async_all in [false, true] { + let output = TempDir::new().unwrap(); + Codegen::new() + .out_dir(output.path()) + .input(proto_input()) + .include(proto_dir()) + .rust_protobuf() + .customize(Customize { + async_all, + ..Default::default() + }) + .run() + .unwrap(); + + let messages = fs::read_to_string(output.path().join("well_known.rs")).unwrap(); + let services = fs::read_to_string(output.path().join("well_known_ttrpc.rs")).unwrap(); + let timestamp = "::protobuf::well_known_types::timestamp::Timestamp"; + let descriptor = "::protobuf::descriptor::FileDescriptorProto"; + let nested_descriptor = "::protobuf::descriptor::descriptor_proto::ExtensionRange"; + + assert!(messages.contains(timestamp)); + assert!(services.contains(timestamp)); + assert!(services.contains(descriptor)); + assert!(services.contains(nested_descriptor)); + assert!(!output.path().join("descriptor.rs").exists()); + assert!(!output.path().join("timestamp.rs").exists()); + } +} + +#[test] +fn missing_non_well_known_import_is_an_error() { + let fixture = TempDir::new().unwrap(); + let input = fixture.path().join("missing.proto"); + fs::write( + &input, + r#"syntax = "proto3"; +import "example/missing.proto"; +message Request {} +"#, + ) + .unwrap(); + + let error = match parse_and_typecheck(&[fixture.path()], &[input.as_path()]) { + Ok(_) => panic!("missing import unexpectedly succeeded"), + Err(error) => error, + }; + assert!(error.to_string().contains("example/missing.proto")); +}