Skip to main content

rocket/data/
data_stream.rs

1use std::pin::Pin;
2use std::task::{Context, Poll};
3use std::path::Path;
4use std::io::{self, Cursor};
5
6use futures::ready;
7use futures::stream::Stream;
8use tokio::fs::File;
9use tokio::io::{AsyncRead, AsyncWrite, AsyncReadExt, ReadBuf, Take};
10use tokio_util::io::StreamReader;
11use hyper::body::{Body, Bytes, Incoming as HyperBody};
12
13use crate::data::{Capped, N};
14use crate::data::transform::Transform;
15use crate::util::Chain;
16
17use super::peekable::Peekable;
18use super::transform::TransformBuf;
19
20/// Raw data stream of a request body.
21///
22/// This stream can only be obtained by calling
23/// [`Data::open()`](crate::data::Data::open()) with a data limit. The stream
24/// contains all of the data in the body of the request.
25///
26/// Reading from a `DataStream` is accomplished via the various methods on the
27/// structure. In general, methods exists in two variants: those that _check_
28/// whether the entire stream was read and those that don't. The former either
29/// directly or indirectly (via [`Capped`]) return an [`N`] which allows
30/// checking if the stream was read to completion while the latter do not.
31///
32/// | Read Into | Method                               | Notes                            |
33/// |-----------|--------------------------------------|----------------------------------|
34/// | `String`  | [`DataStream::into_string()`]        | Completeness checked. Preferred. |
35/// | `String`  | [`AsyncReadExt::read_to_string()`]   | Unchecked w/existing `String`.   |
36/// | `Vec<u8>` | [`DataStream::into_bytes()`]         | Checked. Preferred.              |
37/// | `Vec<u8>` | [`DataStream::stream_to(&mut vec)`]  | Checked w/existing `Vec`.        |
38/// | `Vec<u8>` | [`DataStream::stream_precise_to()`]  | Unchecked w/existing `Vec`.      |
39/// | `File`    | [`DataStream::into_file()`]          | Checked. Preferred.              |
40/// | `File`    | [`DataStream::stream_to(&mut file)`] | Checked w/ existing `File`.      |
41/// | `File`    | [`DataStream::stream_precise_to()`]  | Unchecked w/ existing `File`.    |
42/// | `T`       | [`DataStream::stream_to()`]          | Checked. Any `T: AsyncWrite`.    |
43/// | `T`       | [`DataStream::stream_precise_to()`]  | Unchecked. Any `T: AsyncWrite`.  |
44///
45/// [`DataStream::stream_to(&mut vec)`]: DataStream::stream_to()
46/// [`DataStream::stream_to(&mut file)`]: DataStream::stream_to()
47#[non_exhaustive]
48pub enum DataStream<'r> {
49    #[doc(hidden)]
50    Base(BaseReader<'r>),
51    #[doc(hidden)]
52    Transform(TransformReader<'r>),
53}
54
55/// A data stream that has a `transformer` applied to it.
56pub struct TransformReader<'r> {
57    transformer: Pin<Box<dyn Transform + Send + Sync + 'r>>,
58    stream: Pin<Box<DataStream<'r>>>,
59    inner_done: bool,
60}
61
62/// Limited, pre-buffered reader to the underlying data stream.
63pub type BaseReader<'r> = Take<Chain<Cursor<Vec<u8>>, RawReader<'r>>>;
64
65/// Direct reader to the underlying data stream. Not limited in any manner.
66pub type RawReader<'r> = StreamReader<RawStream<'r>, Bytes>;
67
68/// Raw underlying data stream.
69pub enum RawStream<'r> {
70    Empty,
71    Body(HyperBody),
72    #[cfg(feature = "http3-preview")]
73    H3Body(crate::listener::Cancellable<crate::listener::quic::QuicRx>),
74    Multipart(multer::Field<'r>),
75    /// A read from the stream failed. Some streams report EOF after an error,
76    /// so the error is reported again by every subsequent read.
77    Failed(io::ErrorKind, String),
78}
79
80impl<'r> TransformReader<'r> {
81    /// Returns the underlying `BaseReader`.
82    fn base_mut(&mut self) -> &mut BaseReader<'r> {
83        match self.stream.as_mut().get_mut() {
84            DataStream::Base(base) => base,
85            DataStream::Transform(inner) => inner.base_mut(),
86        }
87    }
88
89    /// Returns the underlying `BaseReader`.
90    fn base(&self) -> &BaseReader<'r> {
91        match self.stream.as_ref().get_ref() {
92            DataStream::Base(base) => base,
93            DataStream::Transform(inner) => inner.base(),
94        }
95    }
96}
97
98impl<'r> DataStream<'r> {
99    pub(crate) fn new(
100        transformers: Vec<Pin<Box<dyn Transform + Send + Sync + 'r>>>,
101        Peekable { buffer, reader, .. }: Peekable<512, RawReader<'r>>,
102        limit: u64
103    ) -> Self {
104        let mut stream = DataStream::Base(Chain::new(Cursor::new(buffer), reader).take(limit));
105        for transformer in transformers {
106            stream = DataStream::Transform(TransformReader {
107                transformer,
108                stream: Box::pin(stream),
109                inner_done: false,
110            });
111        }
112
113        stream
114    }
115
116    /// Returns the underlying `BaseReader`.
117    fn base_mut(&mut self) -> &mut BaseReader<'r> {
118        match self {
119            DataStream::Base(base) => base,
120            DataStream::Transform(transform) => transform.base_mut(),
121        }
122    }
123
124    /// Returns the underlying `BaseReader`.
125    fn base(&self) -> &BaseReader<'r> {
126        match self {
127            DataStream::Base(base) => base,
128            DataStream::Transform(transform) => transform.base(),
129        }
130    }
131
132    /// Whether a previous read exhausted the set limit _and then some_.
133    async fn limit_exceeded(&mut self) -> io::Result<bool> {
134        let base = self.base_mut();
135
136        #[cold]
137        async fn _limit_exceeded(base: &mut BaseReader<'_>) -> io::Result<bool> {
138            // Read one more byte after reaching limit to see if we cut early.
139            base.set_limit(1);
140            let mut buf = [0u8; 1];
141            let exceeded = base.read(&mut buf).await? != 0;
142            base.set_limit(0);
143            Ok(exceeded)
144        }
145
146        Ok(base.limit() == 0 && _limit_exceeded(base).await?)
147    }
148
149    /// Number of bytes a full read from `self` will _definitely_ read.
150    ///
151    /// # Example
152    ///
153    /// ```rust
154    /// use rocket::data::{Data, ToByteUnit};
155    ///
156    /// async fn f(data: Data<'_>) {
157    ///     let definitely_have_n_bytes = data.open(1.kibibytes()).hint();
158    /// }
159    /// ```
160    pub fn hint(&self) -> usize {
161        let base = self.base();
162        if let (Some(cursor), _) = base.get_ref().get_ref() {
163            let len = cursor.get_ref().len() as u64;
164            let position = cursor.position().min(len);
165            let remaining = len - position;
166            remaining.min(base.limit()) as usize
167        } else {
168            0
169        }
170    }
171
172    /// A helper method to write the body of the request to any `AsyncWrite`
173    /// type. Returns an [`N`] which indicates how many bytes were written and
174    /// whether the entire stream was read. An additional read from `self` may
175    /// be required to check if all of the stream has been read. If that
176    /// information is not needed, use [`DataStream::stream_precise_to()`].
177    ///
178    /// This method is identical to `tokio::io::copy(&mut self, &mut writer)`
179    /// except in that it returns an `N` to check for completeness.
180    ///
181    /// # Example
182    ///
183    /// ```rust
184    /// use std::io;
185    /// use rocket::data::{Data, ToByteUnit};
186    ///
187    /// async fn data_guard(mut data: Data<'_>) -> io::Result<String> {
188    ///     // write all of the data to stdout
189    ///     let written = data.open(512.kibibytes())
190    ///         .stream_to(tokio::io::stdout()).await?;
191    ///
192    ///     Ok(format!("Wrote {} bytes.", written))
193    /// }
194    /// ```
195    #[inline(always)]
196    pub async fn stream_to<W>(mut self, mut writer: W) -> io::Result<N>
197        where W: AsyncWrite + Unpin
198    {
199        let written = tokio::io::copy(&mut self, &mut writer).await?;
200        Ok(N { written, complete: !self.limit_exceeded().await? })
201    }
202
203    /// Like [`DataStream::stream_to()`] except that no end-of-stream check is
204    /// conducted and thus read/write completeness is unknown.
205    ///
206    /// # Example
207    ///
208    /// ```rust
209    /// use std::io;
210    /// use rocket::data::{Data, ToByteUnit};
211    ///
212    /// async fn data_guard(mut data: Data<'_>) -> io::Result<String> {
213    ///     // write all of the data to stdout
214    ///     let written = data.open(512.kibibytes())
215    ///         .stream_precise_to(tokio::io::stdout()).await?;
216    ///
217    ///     Ok(format!("Wrote {} bytes.", written))
218    /// }
219    /// ```
220    #[inline(always)]
221    pub async fn stream_precise_to<W>(mut self, mut writer: W) -> io::Result<u64>
222        where W: AsyncWrite + Unpin
223    {
224        tokio::io::copy(&mut self, &mut writer).await
225    }
226
227    /// A helper method to write the body of the request to a `Vec<u8>`.
228    ///
229    /// # Example
230    ///
231    /// ```rust
232    /// use std::io;
233    /// use rocket::data::{Data, ToByteUnit};
234    ///
235    /// async fn data_guard(data: Data<'_>) -> io::Result<Vec<u8>> {
236    ///     let bytes = data.open(4.kibibytes()).into_bytes().await?;
237    ///     if !bytes.is_complete() {
238    ///         println!("there are bytes remaining in the stream");
239    ///     }
240    ///
241    ///     Ok(bytes.into_inner())
242    /// }
243    /// ```
244    pub async fn into_bytes(self) -> io::Result<Capped<Vec<u8>>> {
245        let mut vec = Vec::with_capacity(self.hint());
246        let n = self.stream_to(&mut vec).await?;
247        Ok(Capped { value: vec, n })
248    }
249
250    /// A helper method to write the body of the request to a `String`.
251    ///
252    /// # Example
253    ///
254    /// ```rust
255    /// use std::io;
256    /// use rocket::data::{Data, ToByteUnit};
257    ///
258    /// async fn data_guard(data: Data<'_>) -> io::Result<String> {
259    ///     let string = data.open(10.bytes()).into_string().await?;
260    ///     if !string.is_complete() {
261    ///         println!("there are bytes remaining in the stream");
262    ///     }
263    ///
264    ///     Ok(string.into_inner())
265    /// }
266    /// ```
267    pub async fn into_string(mut self) -> io::Result<Capped<String>> {
268        let mut string = String::with_capacity(self.hint());
269        let written = self.read_to_string(&mut string).await?;
270        let n = N { written: written as u64, complete: !self.limit_exceeded().await? };
271        Ok(Capped { value: string, n })
272    }
273
274    /// A helper method to write the body of the request to a file at the path
275    /// determined by `path`. If a file at the path already exists, it is
276    /// overwritten. The opened file is returned.
277    ///
278    /// # Example
279    ///
280    /// ```rust
281    /// use std::io;
282    /// use rocket::data::{Data, ToByteUnit};
283    ///
284    /// async fn data_guard(mut data: Data<'_>) -> io::Result<String> {
285    ///     let file = data.open(1.megabytes()).into_file("/static/file").await?;
286    ///     if !file.is_complete() {
287    ///         println!("there are bytes remaining in the stream");
288    ///     }
289    ///
290    ///     Ok(format!("Wrote {} bytes to /static/file", file.n))
291    /// }
292    /// ```
293    pub async fn into_file<P: AsRef<Path>>(self, path: P) -> io::Result<Capped<File>> {
294        let mut file = File::create(path).await?;
295        let n = self.stream_to(&mut tokio::io::BufWriter::new(&mut file)).await?;
296        Ok(Capped { value: file, n })
297    }
298}
299
300impl AsyncRead for DataStream<'_> {
301    fn poll_read(
302        self: Pin<&mut Self>,
303        cx: &mut Context<'_>,
304        buf: &mut ReadBuf<'_>,
305    ) -> Poll<io::Result<()>> {
306        match self.get_mut() {
307            DataStream::Base(inner) => Pin::new(inner).poll_read(cx, buf),
308            DataStream::Transform(inner) => Pin::new(inner).poll_read(cx, buf),
309        }
310    }
311}
312
313impl AsyncRead for TransformReader<'_> {
314    fn poll_read(
315        mut self: Pin<&mut Self>,
316        cx: &mut Context<'_>,
317        buf: &mut ReadBuf<'_>,
318    ) -> Poll<io::Result<()>> {
319        let init_fill = buf.filled().len();
320        if !self.inner_done {
321            ready!(Pin::new(&mut self.stream).poll_read(cx, buf))?;
322            self.inner_done = init_fill == buf.filled().len();
323        }
324
325        if self.inner_done {
326            return self.transformer.as_mut().poll_finish(cx, buf);
327        }
328
329        let mut tbuf = TransformBuf { buf, cursor: init_fill };
330        self.transformer.as_mut().transform(&mut tbuf)?;
331        if buf.filled().len() == init_fill {
332            cx.waker().wake_by_ref();
333            return Poll::Pending;
334        }
335
336        Poll::Ready(Ok(()))
337    }
338}
339
340impl Stream for RawStream<'_> {
341    type Item = io::Result<Bytes>;
342
343    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
344        let this = self.get_mut();
345        let next = match this {
346            // TODO: Expose trailer headers, somehow.
347            RawStream::Body(body) => {
348                Pin::new(body)
349                    .poll_frame(cx)
350                    .map_ok(|frame| frame.into_data().unwrap_or_else(|_| Bytes::new()))
351                    .map_err(io::Error::other)
352            },
353            #[cfg(feature = "http3-preview")]
354            RawStream::H3Body(stream) => Pin::new(stream).poll_next(cx),
355            RawStream::Multipart(s) => Pin::new(s).poll_next(cx).map_err(io::Error::other),
356            RawStream::Failed(kind, msg) => {
357                return Poll::Ready(Some(Err(io::Error::new(*kind, msg.clone()))));
358            },
359            RawStream::Empty => Poll::Ready(None),
360        };
361
362        if let Poll::Ready(Some(Err(e))) = &next {
363            *this = RawStream::Failed(e.kind(), e.to_string());
364        }
365
366        next
367    }
368
369    fn size_hint(&self) -> (usize, Option<usize>) {
370        match self {
371            RawStream::Body(body) => {
372                let hint = body.size_hint();
373                let (lower, upper) = (hint.lower(), hint.upper());
374                (lower as usize, upper.map(|x| x as usize))
375            },
376            #[cfg(feature = "http3-preview")]
377            RawStream::H3Body(_) => (0, Some(0)),
378            RawStream::Multipart(mp) => mp.size_hint(),
379            RawStream::Failed(..) | RawStream::Empty => (0, Some(0)),
380        }
381    }
382}
383
384impl std::fmt::Display for RawStream<'_> {
385    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
386        match self {
387            RawStream::Empty => f.write_str("empty stream"),
388            RawStream::Body(_) => f.write_str("request body"),
389            #[cfg(feature = "http3-preview")]
390            RawStream::H3Body(_) => f.write_str("http3 quic stream"),
391            RawStream::Multipart(_) => f.write_str("multipart form field"),
392            RawStream::Failed(..) => f.write_str("failed stream"),
393        }
394    }
395}
396
397impl<'r> From<HyperBody> for RawStream<'r> {
398    fn from(value: HyperBody) -> Self {
399        Self::Body(value)
400    }
401}
402
403#[cfg(feature = "http3-preview")]
404impl<'r> From<crate::listener::Cancellable<crate::listener::quic::QuicRx>> for RawStream<'r> {
405    fn from(value: crate::listener::Cancellable<crate::listener::quic::QuicRx>) -> Self {
406        Self::H3Body(value)
407    }
408}
409
410impl<'r> From<multer::Field<'r>> for RawStream<'r> {
411    fn from(value: multer::Field<'r>) -> Self {
412        Self::Multipart(value)
413    }
414}