rocket/data/data_stream.rs
1use std::pin::Pin;
2use std::task::{Context, Poll};
3use std::path::Path;
4use std::io::{self, Cursor};
5
6use futures::ready;
7use futures::stream::Stream;
8use tokio::fs::File;
9use tokio::io::{AsyncRead, AsyncWrite, AsyncReadExt, ReadBuf, Take};
10use tokio_util::io::StreamReader;
11use hyper::body::{Body, Bytes, Incoming as HyperBody};
12
13use crate::data::{Capped, N};
14use crate::data::transform::Transform;
15use crate::util::Chain;
16
17use super::peekable::Peekable;
18use super::transform::TransformBuf;
19
20/// Raw data stream of a request body.
21///
22/// This stream can only be obtained by calling
23/// [`Data::open()`](crate::data::Data::open()) with a data limit. The stream
24/// contains all of the data in the body of the request.
25///
26/// Reading from a `DataStream` is accomplished via the various methods on the
27/// structure. In general, methods exists in two variants: those that _check_
28/// whether the entire stream was read and those that don't. The former either
29/// directly or indirectly (via [`Capped`]) return an [`N`] which allows
30/// checking if the stream was read to completion while the latter do not.
31///
32/// | Read Into | Method | Notes |
33/// |-----------|--------------------------------------|----------------------------------|
34/// | `String` | [`DataStream::into_string()`] | Completeness checked. Preferred. |
35/// | `String` | [`AsyncReadExt::read_to_string()`] | Unchecked w/existing `String`. |
36/// | `Vec<u8>` | [`DataStream::into_bytes()`] | Checked. Preferred. |
37/// | `Vec<u8>` | [`DataStream::stream_to(&mut vec)`] | Checked w/existing `Vec`. |
38/// | `Vec<u8>` | [`DataStream::stream_precise_to()`] | Unchecked w/existing `Vec`. |
39/// | `File` | [`DataStream::into_file()`] | Checked. Preferred. |
40/// | `File` | [`DataStream::stream_to(&mut file)`] | Checked w/ existing `File`. |
41/// | `File` | [`DataStream::stream_precise_to()`] | Unchecked w/ existing `File`. |
42/// | `T` | [`DataStream::stream_to()`] | Checked. Any `T: AsyncWrite`. |
43/// | `T` | [`DataStream::stream_precise_to()`] | Unchecked. Any `T: AsyncWrite`. |
44///
45/// [`DataStream::stream_to(&mut vec)`]: DataStream::stream_to()
46/// [`DataStream::stream_to(&mut file)`]: DataStream::stream_to()
47#[non_exhaustive]
48pub enum DataStream<'r> {
49 #[doc(hidden)]
50 Base(BaseReader<'r>),
51 #[doc(hidden)]
52 Transform(TransformReader<'r>),
53}
54
55/// A data stream that has a `transformer` applied to it.
56pub struct TransformReader<'r> {
57 transformer: Pin<Box<dyn Transform + Send + Sync + 'r>>,
58 stream: Pin<Box<DataStream<'r>>>,
59 inner_done: bool,
60}
61
62/// Limited, pre-buffered reader to the underlying data stream.
63pub type BaseReader<'r> = Take<Chain<Cursor<Vec<u8>>, RawReader<'r>>>;
64
65/// Direct reader to the underlying data stream. Not limited in any manner.
66pub type RawReader<'r> = StreamReader<RawStream<'r>, Bytes>;
67
68/// Raw underlying data stream.
69pub enum RawStream<'r> {
70 Empty,
71 Body(HyperBody),
72 #[cfg(feature = "http3-preview")]
73 H3Body(crate::listener::Cancellable<crate::listener::quic::QuicRx>),
74 Multipart(multer::Field<'r>),
75 /// A read from the stream failed. Some streams report EOF after an error,
76 /// so the error is reported again by every subsequent read.
77 Failed(io::ErrorKind, String),
78}
79
80impl<'r> TransformReader<'r> {
81 /// Returns the underlying `BaseReader`.
82 fn base_mut(&mut self) -> &mut BaseReader<'r> {
83 match self.stream.as_mut().get_mut() {
84 DataStream::Base(base) => base,
85 DataStream::Transform(inner) => inner.base_mut(),
86 }
87 }
88
89 /// Returns the underlying `BaseReader`.
90 fn base(&self) -> &BaseReader<'r> {
91 match self.stream.as_ref().get_ref() {
92 DataStream::Base(base) => base,
93 DataStream::Transform(inner) => inner.base(),
94 }
95 }
96}
97
98impl<'r> DataStream<'r> {
99 pub(crate) fn new(
100 transformers: Vec<Pin<Box<dyn Transform + Send + Sync + 'r>>>,
101 Peekable { buffer, reader, .. }: Peekable<512, RawReader<'r>>,
102 limit: u64
103 ) -> Self {
104 let mut stream = DataStream::Base(Chain::new(Cursor::new(buffer), reader).take(limit));
105 for transformer in transformers {
106 stream = DataStream::Transform(TransformReader {
107 transformer,
108 stream: Box::pin(stream),
109 inner_done: false,
110 });
111 }
112
113 stream
114 }
115
116 /// Returns the underlying `BaseReader`.
117 fn base_mut(&mut self) -> &mut BaseReader<'r> {
118 match self {
119 DataStream::Base(base) => base,
120 DataStream::Transform(transform) => transform.base_mut(),
121 }
122 }
123
124 /// Returns the underlying `BaseReader`.
125 fn base(&self) -> &BaseReader<'r> {
126 match self {
127 DataStream::Base(base) => base,
128 DataStream::Transform(transform) => transform.base(),
129 }
130 }
131
132 /// Whether a previous read exhausted the set limit _and then some_.
133 async fn limit_exceeded(&mut self) -> io::Result<bool> {
134 let base = self.base_mut();
135
136 #[cold]
137 async fn _limit_exceeded(base: &mut BaseReader<'_>) -> io::Result<bool> {
138 // Read one more byte after reaching limit to see if we cut early.
139 base.set_limit(1);
140 let mut buf = [0u8; 1];
141 let exceeded = base.read(&mut buf).await? != 0;
142 base.set_limit(0);
143 Ok(exceeded)
144 }
145
146 Ok(base.limit() == 0 && _limit_exceeded(base).await?)
147 }
148
149 /// Number of bytes a full read from `self` will _definitely_ read.
150 ///
151 /// # Example
152 ///
153 /// ```rust
154 /// use rocket::data::{Data, ToByteUnit};
155 ///
156 /// async fn f(data: Data<'_>) {
157 /// let definitely_have_n_bytes = data.open(1.kibibytes()).hint();
158 /// }
159 /// ```
160 pub fn hint(&self) -> usize {
161 let base = self.base();
162 if let (Some(cursor), _) = base.get_ref().get_ref() {
163 let len = cursor.get_ref().len() as u64;
164 let position = cursor.position().min(len);
165 let remaining = len - position;
166 remaining.min(base.limit()) as usize
167 } else {
168 0
169 }
170 }
171
172 /// A helper method to write the body of the request to any `AsyncWrite`
173 /// type. Returns an [`N`] which indicates how many bytes were written and
174 /// whether the entire stream was read. An additional read from `self` may
175 /// be required to check if all of the stream has been read. If that
176 /// information is not needed, use [`DataStream::stream_precise_to()`].
177 ///
178 /// This method is identical to `tokio::io::copy(&mut self, &mut writer)`
179 /// except in that it returns an `N` to check for completeness.
180 ///
181 /// # Example
182 ///
183 /// ```rust
184 /// use std::io;
185 /// use rocket::data::{Data, ToByteUnit};
186 ///
187 /// async fn data_guard(mut data: Data<'_>) -> io::Result<String> {
188 /// // write all of the data to stdout
189 /// let written = data.open(512.kibibytes())
190 /// .stream_to(tokio::io::stdout()).await?;
191 ///
192 /// Ok(format!("Wrote {} bytes.", written))
193 /// }
194 /// ```
195 #[inline(always)]
196 pub async fn stream_to<W>(mut self, mut writer: W) -> io::Result<N>
197 where W: AsyncWrite + Unpin
198 {
199 let written = tokio::io::copy(&mut self, &mut writer).await?;
200 Ok(N { written, complete: !self.limit_exceeded().await? })
201 }
202
203 /// Like [`DataStream::stream_to()`] except that no end-of-stream check is
204 /// conducted and thus read/write completeness is unknown.
205 ///
206 /// # Example
207 ///
208 /// ```rust
209 /// use std::io;
210 /// use rocket::data::{Data, ToByteUnit};
211 ///
212 /// async fn data_guard(mut data: Data<'_>) -> io::Result<String> {
213 /// // write all of the data to stdout
214 /// let written = data.open(512.kibibytes())
215 /// .stream_precise_to(tokio::io::stdout()).await?;
216 ///
217 /// Ok(format!("Wrote {} bytes.", written))
218 /// }
219 /// ```
220 #[inline(always)]
221 pub async fn stream_precise_to<W>(mut self, mut writer: W) -> io::Result<u64>
222 where W: AsyncWrite + Unpin
223 {
224 tokio::io::copy(&mut self, &mut writer).await
225 }
226
227 /// A helper method to write the body of the request to a `Vec<u8>`.
228 ///
229 /// # Example
230 ///
231 /// ```rust
232 /// use std::io;
233 /// use rocket::data::{Data, ToByteUnit};
234 ///
235 /// async fn data_guard(data: Data<'_>) -> io::Result<Vec<u8>> {
236 /// let bytes = data.open(4.kibibytes()).into_bytes().await?;
237 /// if !bytes.is_complete() {
238 /// println!("there are bytes remaining in the stream");
239 /// }
240 ///
241 /// Ok(bytes.into_inner())
242 /// }
243 /// ```
244 pub async fn into_bytes(self) -> io::Result<Capped<Vec<u8>>> {
245 let mut vec = Vec::with_capacity(self.hint());
246 let n = self.stream_to(&mut vec).await?;
247 Ok(Capped { value: vec, n })
248 }
249
250 /// A helper method to write the body of the request to a `String`.
251 ///
252 /// # Example
253 ///
254 /// ```rust
255 /// use std::io;
256 /// use rocket::data::{Data, ToByteUnit};
257 ///
258 /// async fn data_guard(data: Data<'_>) -> io::Result<String> {
259 /// let string = data.open(10.bytes()).into_string().await?;
260 /// if !string.is_complete() {
261 /// println!("there are bytes remaining in the stream");
262 /// }
263 ///
264 /// Ok(string.into_inner())
265 /// }
266 /// ```
267 pub async fn into_string(mut self) -> io::Result<Capped<String>> {
268 let mut string = String::with_capacity(self.hint());
269 let written = self.read_to_string(&mut string).await?;
270 let n = N { written: written as u64, complete: !self.limit_exceeded().await? };
271 Ok(Capped { value: string, n })
272 }
273
274 /// A helper method to write the body of the request to a file at the path
275 /// determined by `path`. If a file at the path already exists, it is
276 /// overwritten. The opened file is returned.
277 ///
278 /// # Example
279 ///
280 /// ```rust
281 /// use std::io;
282 /// use rocket::data::{Data, ToByteUnit};
283 ///
284 /// async fn data_guard(mut data: Data<'_>) -> io::Result<String> {
285 /// let file = data.open(1.megabytes()).into_file("/static/file").await?;
286 /// if !file.is_complete() {
287 /// println!("there are bytes remaining in the stream");
288 /// }
289 ///
290 /// Ok(format!("Wrote {} bytes to /static/file", file.n))
291 /// }
292 /// ```
293 pub async fn into_file<P: AsRef<Path>>(self, path: P) -> io::Result<Capped<File>> {
294 let mut file = File::create(path).await?;
295 let n = self.stream_to(&mut tokio::io::BufWriter::new(&mut file)).await?;
296 Ok(Capped { value: file, n })
297 }
298}
299
300impl AsyncRead for DataStream<'_> {
301 fn poll_read(
302 self: Pin<&mut Self>,
303 cx: &mut Context<'_>,
304 buf: &mut ReadBuf<'_>,
305 ) -> Poll<io::Result<()>> {
306 match self.get_mut() {
307 DataStream::Base(inner) => Pin::new(inner).poll_read(cx, buf),
308 DataStream::Transform(inner) => Pin::new(inner).poll_read(cx, buf),
309 }
310 }
311}
312
313impl AsyncRead for TransformReader<'_> {
314 fn poll_read(
315 mut self: Pin<&mut Self>,
316 cx: &mut Context<'_>,
317 buf: &mut ReadBuf<'_>,
318 ) -> Poll<io::Result<()>> {
319 let init_fill = buf.filled().len();
320 if !self.inner_done {
321 ready!(Pin::new(&mut self.stream).poll_read(cx, buf))?;
322 self.inner_done = init_fill == buf.filled().len();
323 }
324
325 if self.inner_done {
326 return self.transformer.as_mut().poll_finish(cx, buf);
327 }
328
329 let mut tbuf = TransformBuf { buf, cursor: init_fill };
330 self.transformer.as_mut().transform(&mut tbuf)?;
331 if buf.filled().len() == init_fill {
332 cx.waker().wake_by_ref();
333 return Poll::Pending;
334 }
335
336 Poll::Ready(Ok(()))
337 }
338}
339
340impl Stream for RawStream<'_> {
341 type Item = io::Result<Bytes>;
342
343 fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
344 let this = self.get_mut();
345 let next = match this {
346 // TODO: Expose trailer headers, somehow.
347 RawStream::Body(body) => {
348 Pin::new(body)
349 .poll_frame(cx)
350 .map_ok(|frame| frame.into_data().unwrap_or_else(|_| Bytes::new()))
351 .map_err(io::Error::other)
352 },
353 #[cfg(feature = "http3-preview")]
354 RawStream::H3Body(stream) => Pin::new(stream).poll_next(cx),
355 RawStream::Multipart(s) => Pin::new(s).poll_next(cx).map_err(io::Error::other),
356 RawStream::Failed(kind, msg) => {
357 return Poll::Ready(Some(Err(io::Error::new(*kind, msg.clone()))));
358 },
359 RawStream::Empty => Poll::Ready(None),
360 };
361
362 if let Poll::Ready(Some(Err(e))) = &next {
363 *this = RawStream::Failed(e.kind(), e.to_string());
364 }
365
366 next
367 }
368
369 fn size_hint(&self) -> (usize, Option<usize>) {
370 match self {
371 RawStream::Body(body) => {
372 let hint = body.size_hint();
373 let (lower, upper) = (hint.lower(), hint.upper());
374 (lower as usize, upper.map(|x| x as usize))
375 },
376 #[cfg(feature = "http3-preview")]
377 RawStream::H3Body(_) => (0, Some(0)),
378 RawStream::Multipart(mp) => mp.size_hint(),
379 RawStream::Failed(..) | RawStream::Empty => (0, Some(0)),
380 }
381 }
382}
383
384impl std::fmt::Display for RawStream<'_> {
385 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
386 match self {
387 RawStream::Empty => f.write_str("empty stream"),
388 RawStream::Body(_) => f.write_str("request body"),
389 #[cfg(feature = "http3-preview")]
390 RawStream::H3Body(_) => f.write_str("http3 quic stream"),
391 RawStream::Multipart(_) => f.write_str("multipart form field"),
392 RawStream::Failed(..) => f.write_str("failed stream"),
393 }
394 }
395}
396
397impl<'r> From<HyperBody> for RawStream<'r> {
398 fn from(value: HyperBody) -> Self {
399 Self::Body(value)
400 }
401}
402
403#[cfg(feature = "http3-preview")]
404impl<'r> From<crate::listener::Cancellable<crate::listener::quic::QuicRx>> for RawStream<'r> {
405 fn from(value: crate::listener::Cancellable<crate::listener::quic::QuicRx>) -> Self {
406 Self::H3Body(value)
407 }
408}
409
410impl<'r> From<multer::Field<'r>> for RawStream<'r> {
411 fn from(value: multer::Field<'r>) -> Self {
412 Self::Multipart(value)
413 }
414}