libproc_macro: Add TokenStream implementation

Add the TokenStream rust type implementation to libproc_macro. Also
implement IntoIter structure to iterator over a TokenStream.

ChangeLog:

	* librust/proc_macro/rust/bridge.rs: Add group and token_stream
	modules.
	* librust/proc_macro/rust/group.rs: Add group internal
	representation.
	* librust/proc_macro/rust/lib.rs: Add TokenStream
	implementation.
	* librust/proc_macro/rust/literal.rs: Make internal
	implementation visible in the whole crate.
	* librust/proc_macro/rust/token_stream.rs: Implement IntoIter.
	* librust/proc_macro/rust/bridge/group.rs: Add Group internals.
	* librust/proc_macro/rust/bridge/token_stream.rs: Add
	TokenStream internals.

Signed-off-by: Pierre-Emmanuel Patry <pierre-emmanuel.patry@embecosm.com>

7 files changed, 89 insertions, 14 deletions

diff --git a/librust/proc_macro/rust/bridge.rs b/librust/proc_macro/rust/bridge.rs
index ff2c496..eb7f854 100644
--- a/librust/proc_macro/rust/bridge.rs
+++ b/librust/proc_macro/rust/bridge.rs
@@ -1,7 +1,9 @@
+pub mod group;
 pub mod ident;
 pub mod literal;
 pub mod punct;
 pub mod span;
+pub mod token_stream;
 
 extern "C" {
     fn bridge__is_available() -> bool;
diff --git a/librust/proc_macro/rust/bridge/group.rs b/librust/proc_macro/rust/bridge/group.rs
new file mode 100644
index 0000000..6590f89
--- /dev/null
+++ b/librust/proc_macro/rust/bridge/group.rs
@@ -0,0 +1,9 @@
+use bridge;
+use Delimiter;
+
+#[repr(C)]
+#[derive(Clone)]
+pub struct Group {
+    delimiter: Delimiter,
+    stream: bridge::token_stream::TokenStream,
+}
diff --git a/librust/proc_macro/rust/bridge/token_stream.rs b/librust/proc_macro/rust/bridge/token_stream.rs
new file mode 100644
index 0000000..307ff6b
--- /dev/null
+++ b/librust/proc_macro/rust/bridge/token_stream.rs
@@ -0,0 +1,35 @@
+use bridge::{group::Group, ident::Ident, literal::Literal, punct::Punct};
+
+// TODO: There surely is a better way to achieve this. I don't like this
+// "duplication" of the TokenTree enumeration. But I cannot use the public
+// one since it's underlying types (public Group, Ident...) are not ffi safe.
+// Flattening all those struct might be a solution.
+#[repr(C)]
+#[derive(Clone)]
+pub enum TokenTree {
+    Group(Group),
+    Ident(Ident),
+    Punct(Punct),
+    Literal(Literal),
+}
+
+extern "C" {
+    fn TokenStream__new() -> TokenStream;
+}
+
+#[repr(C)]
+#[derive(Clone)]
+pub struct TokenStream {
+    pub(crate) data: *const TokenTree,
+    pub(crate) size: u64,
+}
+
+impl TokenStream {
+    pub fn new() -> Self {
+        unsafe { TokenStream__new() }
+    }
+
+    pub fn is_empty(&self) -> bool {
+        0 == self.size
+    }
+}
diff --git a/librust/proc_macro/rust/group.rs b/librust/proc_macro/rust/group.rs
index e0e3de9..6512982 100644
--- a/librust/proc_macro/rust/group.rs
+++ b/librust/proc_macro/rust/group.rs
@@ -1,8 +1,10 @@
+use bridge;
 use std::fmt;
 use Span;
 use TokenStream;
 
 /// Describes how a sequence of token trees is delimited.
+#[repr(C)]
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
 pub enum Delimiter {
     /// The sequence is delimited by a parenthesis `(...)`.
@@ -17,9 +19,7 @@ pub enum Delimiter {
 
 /// A delimited token stream.
 #[derive(Clone)]
-pub struct Group {
-    // Internal implementation details
-}
+pub struct Group(pub(crate) bridge::group::Group);
 
 impl Group {
     /// Creates a new `Group`.
diff --git a/librust/proc_macro/rust/lib.rs b/librust/proc_macro/rust/lib.rs
index 4e6802a..31aa6ea 100644
--- a/librust/proc_macro/rust/lib.rs
+++ b/librust/proc_macro/rust/lib.rs
@@ -34,6 +34,19 @@ pub enum TokenTree {
     Literal(Literal),
 }
 
+type InternalTokenTree = bridge::token_stream::TokenTree;
+
+impl From<InternalTokenTree> for TokenTree {
+    fn from(value: InternalTokenTree) -> Self {
+        match value {
+            InternalTokenTree::Group(g) => TokenTree::Group(Group(g)),
+            InternalTokenTree::Ident(i) => TokenTree::Ident(Ident(i)),
+            InternalTokenTree::Punct(p) => TokenTree::Punct(Punct(p)),
+            InternalTokenTree::Literal(l) => TokenTree::Literal(Literal(l)),
+        }
+    }
+}
+
 impl TokenTree {
     /// Get the [`Span`] for this TokenTree.
     pub fn span(&self) -> Span {
@@ -124,21 +137,19 @@ impl error::Error for LexError {}
 /// is both the input and the output of `#[proc_macro]`,
 /// `#[proc_macro_attribute]` and `#[proc_macro_derive]` definitions.
 #[derive(Clone)]
-pub struct TokenStream {
-    // Internal implementation details
-}
+pub struct TokenStream(bridge::token_stream::TokenStream);
 
 impl TokenStream {
     // TODO: Add experimental API functions for this type
 
     /// Creates an empty `TokenStream` containing no token trees.
     pub fn new() -> Self {
-        todo!("Implement this function")
+        TokenStream(bridge::token_stream::TokenStream::new())
     }
 
     /// Checks if this `TokenStream` is empty.
     pub fn is_empty(&self) -> bool {
-        todo!("Implement this function")
+        self.0.is_empty()
     }
 }
 
diff --git a/librust/proc_macro/rust/literal.rs b/librust/proc_macro/rust/literal.rs
index 2820da6..4011201 100644
--- a/librust/proc_macro/rust/literal.rs
+++ b/librust/proc_macro/rust/literal.rs
@@ -21,7 +21,7 @@ use Span;
 /// Boolean literals like `true` and `false` are `Ident`s and do not belong
 /// here.
 #[derive(Clone)]
-pub struct Literal(bridge::literal::Literal);
+pub struct Literal(pub(crate) bridge::literal::Literal);
 
 impl Literal {
     // TODO: Add experimental API functions for this type
diff --git a/librust/proc_macro/rust/token_stream.rs b/librust/proc_macro/rust/token_stream.rs
index 3de89df..c4caf9f 100644
--- a/librust/proc_macro/rust/token_stream.rs
+++ b/librust/proc_macro/rust/token_stream.rs
@@ -1,26 +1,40 @@
 //! Public implementation details for the `TokenStream` type, such as iterators.
+use bridge;
+use std::convert::TryInto;
+
 use TokenStream;
 use TokenTree;
 
 /// An iterator over [`TokenStream`]'s [`TokenTree`]s.
 #[derive(Clone)]
 pub struct IntoIter {
-    // Internal implementation details
+    current: *const bridge::token_stream::TokenTree,
+    end: *const bridge::token_stream::TokenTree,
 }
 
 impl Iterator for IntoIter {
     type Item = TokenTree;
 
     fn next(&mut self) -> Option<TokenTree> {
-        todo!("Implement this function")
+        if self.current == self.end {
+            return None;
+        } else {
+            let result = self.current;
+            self.current = unsafe { self.current.add(1) };
+            Some(unsafe { std::ptr::read(result) }.into())
+        }
     }
 
     fn size_hint(&self) -> (usize, Option<usize>) {
-        todo!("Implement this function")
+        // TODO: I'm not a fan of those casts, once #![feature(ptr_sub_ptr)]
+        // is implemented we may replace this line by the following:
+        // self.end.sub_ptr(self.current)
+        let remaining = self.end as usize - self.current as usize;
+        (remaining, Some(remaining))
     }
 
     fn count(self) -> usize {
-        todo!("Implement this function")
+        self.end as usize - self.current as usize
     }
 }
 
@@ -29,6 +43,10 @@ impl IntoIterator for TokenStream {
     type IntoIter = IntoIter;
 
     fn into_iter(self) -> IntoIter {
-        todo!("Implement this function")
+        let capacity = self.0.size.try_into().unwrap();
+        IntoIter {
+            current: self.0.data,
+            end: unsafe { self.0.data.add(capacity) },
+        }
     }
 }