object_store/
parse.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use crate::ObjectStore;
19#[cfg(all(feature = "fs", not(target_arch = "wasm32")))]
20use crate::local::LocalFileSystem;
21use crate::memory::InMemory;
22use crate::path::Path;
23use url::Url;
24
25#[derive(Debug, thiserror::Error)]
26pub enum Error {
27    #[error("Unable to recognise URL \"{}\"", url)]
28    Unrecognised { url: Url },
29
30    #[error(transparent)]
31    Path {
32        #[from]
33        source: crate::path::Error,
34    },
35}
36
37impl From<Error> for super::Error {
38    fn from(e: Error) -> Self {
39        Self::Generic {
40            store: "URL",
41            source: Box::new(e),
42        }
43    }
44}
45
46/// Recognizes various URL formats, identifying the relevant [`ObjectStore`]
47///
48/// See [`ObjectStoreScheme::parse`] for more details
49///
50/// # Supported formats:
51/// - `file:///path/to/my/file` -> [`LocalFileSystem`]
52/// - `memory:///` -> [`InMemory`]
53/// - `s3://bucket/path` -> [`AmazonS3`](crate::aws::AmazonS3) (also supports `s3a`)
54/// - `gs://bucket/path` -> [`GoogleCloudStorage`](crate::gcp::GoogleCloudStorage)
55/// - `[az|abfs[s]]://container[@<account>.<host>]/path` -> [`MicrosoftAzure`](crate::azure::MicrosoftAzure)
56/// - `http://mydomain/path` -> [`HttpStore`](crate::http::HttpStore)
57/// - `https://mydomain/path` -> [`HttpStore`](crate::http::HttpStore)
58///
59/// There are also special cases for AWS and Azure for `https://{host?}/path` paths:
60/// - `dfs.core.windows.net`, `blob.core.windows.net`, `dfs.fabric.microsoft.com`, `blob.fabric.microsoft.com` -> [`MicrosoftAzure`](crate::azure::MicrosoftAzure)
61/// - `amazonaws.com` -> [`AmazonS3`](crate::aws::AmazonS3)
62/// - `r2.cloudflarestorage.com` -> [`AmazonS3`](crate::aws::AmazonS3)
63///
64#[non_exhaustive] // permit new variants
65#[derive(Debug, Eq, PartialEq, Clone)]
66pub enum ObjectStoreScheme {
67    /// Url corresponding to [`LocalFileSystem`]
68    Local,
69    /// Url corresponding to [`InMemory`]
70    Memory,
71    /// Url corresponding to [`AmazonS3`](crate::aws::AmazonS3)
72    AmazonS3,
73    /// Url corresponding to [`GoogleCloudStorage`](crate::gcp::GoogleCloudStorage)
74    GoogleCloudStorage,
75    /// Url corresponding to [`MicrosoftAzure`](crate::azure::MicrosoftAzure)
76    MicrosoftAzure,
77    /// Url corresponding to [`HttpStore`](crate::http::HttpStore)
78    Http,
79}
80
81impl ObjectStoreScheme {
82    /// Create an [`ObjectStoreScheme`] from the provided [`Url`]
83    ///
84    /// Returns the [`ObjectStoreScheme`] and the remaining [`Path`]
85    ///
86    /// # Example
87    /// ```
88    /// # use url::Url;
89    /// # use object_store::ObjectStoreScheme;
90    /// let url: Url = "file:///path/to/my/file".parse().unwrap();
91    /// let (scheme, path) = ObjectStoreScheme::parse(&url).unwrap();
92    /// assert_eq!(scheme, ObjectStoreScheme::Local);
93    /// assert_eq!(path.as_ref(), "path/to/my/file");
94    ///
95    /// let url: Url = "https://blob.core.windows.net/container/path/to/my/file".parse().unwrap();
96    /// let (scheme, path) = ObjectStoreScheme::parse(&url).unwrap();
97    /// assert_eq!(scheme, ObjectStoreScheme::MicrosoftAzure);
98    /// assert_eq!(path.as_ref(), "path/to/my/file");
99    ///
100    /// let url: Url = "https://example.com/path/to/my/file".parse().unwrap();
101    /// let (scheme, path) = ObjectStoreScheme::parse(&url).unwrap();
102    /// assert_eq!(scheme, ObjectStoreScheme::Http);
103    /// assert_eq!(path.as_ref(), "path/to/my/file");
104    /// ```
105    pub fn parse(url: &Url) -> Result<(Self, Path), Error> {
106        let strip_bucket = || Some(url.path().strip_prefix('/')?.split_once('/')?.1);
107
108        let (scheme, path) = match (url.scheme(), url.host_str()) {
109            ("file", None) => (Self::Local, url.path()),
110            ("memory", None) => (Self::Memory, url.path()),
111            ("s3" | "s3a", Some(_)) => (Self::AmazonS3, url.path()),
112            ("gs", Some(_)) => (Self::GoogleCloudStorage, url.path()),
113            ("az" | "adl" | "azure" | "abfs" | "abfss", Some(_)) => {
114                (Self::MicrosoftAzure, url.path())
115            }
116            ("http", Some(_)) => (Self::Http, url.path()),
117            ("https", Some(host)) => {
118                if host.ends_with("dfs.core.windows.net")
119                    || host.ends_with("blob.core.windows.net")
120                    || host.ends_with("dfs.fabric.microsoft.com")
121                    || host.ends_with("blob.fabric.microsoft.com")
122                {
123                    (Self::MicrosoftAzure, strip_bucket().unwrap_or_default())
124                } else if host.ends_with("amazonaws.com") {
125                    match host.starts_with("s3") {
126                        true => (Self::AmazonS3, strip_bucket().unwrap_or_default()),
127                        false => (Self::AmazonS3, url.path()),
128                    }
129                } else if host.ends_with("r2.cloudflarestorage.com") {
130                    (Self::AmazonS3, strip_bucket().unwrap_or_default())
131                } else {
132                    (Self::Http, url.path())
133                }
134            }
135            _ => return Err(Error::Unrecognised { url: url.clone() }),
136        };
137
138        Ok((scheme, Path::from_url_path(path)?))
139    }
140}
141
142#[cfg(feature = "cloud-base")]
143macro_rules! builder_opts {
144    ($builder:ty, $url:expr, $options:expr) => {{
145        let builder = $options.into_iter().fold(
146            <$builder>::new().with_url($url.to_string()),
147            |builder, (key, value)| match key.as_ref().to_ascii_lowercase().parse() {
148                Ok(k) => builder.with_config(k, value),
149                Err(_) => builder,
150            },
151        );
152        Box::new(builder.build()?) as _
153    }};
154}
155
156/// Create an [`ObjectStore`] based on the provided `url`
157///
158/// Returns
159/// - An [`ObjectStore`] of the corresponding type
160/// - The [`Path`] into the [`ObjectStore`] of the addressed resource
161pub fn parse_url(url: &Url) -> Result<(Box<dyn ObjectStore>, Path), super::Error> {
162    parse_url_opts(url, std::iter::empty::<(&str, &str)>())
163}
164
165/// Create an [`ObjectStore`] based on the provided `url` and options
166///
167/// This method can be used to create an instance of one of the provided
168/// `ObjectStore` implementations based on the URL scheme (see
169/// [`ObjectStoreScheme`] for more details).
170///
171/// For example
172/// * `file:///path/to/my/file` will return a [`LocalFileSystem`] instance
173/// * `s3://bucket/path` will return an [`AmazonS3`] instance if the `aws` feature is enabled.
174///
175/// Arguments:
176/// * `url`: The URL to parse
177/// * `options`: A list of key-value pairs to pass to the [`ObjectStore`] builder.
178///   Note different object stores accept different configuration options, so
179///   the options that are read depends on the `url` value. One common pattern
180///   is to pass configuration information via process variables using [`std::env::vars`].
181///
182/// Returns
183/// - An [`ObjectStore`] of the corresponding type
184/// - The [`Path`] into the [`ObjectStore`] of the addressed resource
185///
186/// [`AmazonS3`]: https://docs.rs/object_store/0.12.0/object_store/aws/struct.AmazonS3.html
187pub fn parse_url_opts<I, K, V>(
188    url: &Url,
189    options: I,
190) -> Result<(Box<dyn ObjectStore>, Path), super::Error>
191where
192    I: IntoIterator<Item = (K, V)>,
193    K: AsRef<str>,
194    V: Into<String>,
195{
196    let _options = options;
197    let (scheme, path) = ObjectStoreScheme::parse(url)?;
198    let path = Path::parse(path)?;
199
200    let store = match scheme {
201        #[cfg(all(feature = "fs", not(target_arch = "wasm32")))]
202        ObjectStoreScheme::Local => Box::new(LocalFileSystem::new()) as _,
203        ObjectStoreScheme::Memory => Box::new(InMemory::new()) as _,
204        #[cfg(feature = "aws-base")]
205        ObjectStoreScheme::AmazonS3 => {
206            builder_opts!(crate::aws::AmazonS3Builder, url, _options)
207        }
208        #[cfg(feature = "gcp-base")]
209        ObjectStoreScheme::GoogleCloudStorage => {
210            builder_opts!(crate::gcp::GoogleCloudStorageBuilder, url, _options)
211        }
212        #[cfg(feature = "azure-base")]
213        ObjectStoreScheme::MicrosoftAzure => {
214            builder_opts!(crate::azure::MicrosoftAzureBuilder, url, _options)
215        }
216        #[cfg(feature = "http-base")]
217        ObjectStoreScheme::Http => {
218            let url = &url[..url::Position::BeforePath];
219            builder_opts!(crate::http::HttpBuilder, url, _options)
220        }
221        #[cfg(not(all(
222            feature = "fs",
223            feature = "aws-base",
224            feature = "azure-base",
225            feature = "gcp-base",
226            feature = "http-base",
227            not(target_arch = "wasm32")
228        )))]
229        s => {
230            return Err(super::Error::Generic {
231                store: "parse_url",
232                source: format!("feature for {s:?} not enabled").into(),
233            });
234        }
235    };
236
237    Ok((store, path))
238}
239
240#[cfg(test)]
241mod tests {
242    use super::*;
243    use url::Url;
244
245    #[test]
246    fn test_parse() {
247        let cases = [
248            ("file:/path", (ObjectStoreScheme::Local, "path")),
249            ("file:///path", (ObjectStoreScheme::Local, "path")),
250            ("memory:/path", (ObjectStoreScheme::Memory, "path")),
251            ("memory:///", (ObjectStoreScheme::Memory, "")),
252            ("s3://bucket/path", (ObjectStoreScheme::AmazonS3, "path")),
253            ("s3a://bucket/path", (ObjectStoreScheme::AmazonS3, "path")),
254            (
255                "https://s3.region.amazonaws.com/bucket",
256                (ObjectStoreScheme::AmazonS3, ""),
257            ),
258            (
259                "https://s3.region.amazonaws.com/bucket/path",
260                (ObjectStoreScheme::AmazonS3, "path"),
261            ),
262            (
263                "https://bucket.s3.region.amazonaws.com",
264                (ObjectStoreScheme::AmazonS3, ""),
265            ),
266            (
267                "https://ACCOUNT_ID.r2.cloudflarestorage.com/bucket",
268                (ObjectStoreScheme::AmazonS3, ""),
269            ),
270            (
271                "https://ACCOUNT_ID.r2.cloudflarestorage.com/bucket/path",
272                (ObjectStoreScheme::AmazonS3, "path"),
273            ),
274            (
275                "abfs://container/path",
276                (ObjectStoreScheme::MicrosoftAzure, "path"),
277            ),
278            (
279                "abfs://file_system@account_name.dfs.core.windows.net/path",
280                (ObjectStoreScheme::MicrosoftAzure, "path"),
281            ),
282            (
283                "abfss://file_system@account_name.dfs.core.windows.net/path",
284                (ObjectStoreScheme::MicrosoftAzure, "path"),
285            ),
286            (
287                "https://account.dfs.core.windows.net",
288                (ObjectStoreScheme::MicrosoftAzure, ""),
289            ),
290            (
291                "https://account.dfs.core.windows.net/container/path",
292                (ObjectStoreScheme::MicrosoftAzure, "path"),
293            ),
294            (
295                "https://account.blob.core.windows.net",
296                (ObjectStoreScheme::MicrosoftAzure, ""),
297            ),
298            (
299                "https://account.blob.core.windows.net/container/path",
300                (ObjectStoreScheme::MicrosoftAzure, "path"),
301            ),
302            (
303                "az://container/path",
304                (ObjectStoreScheme::MicrosoftAzure, "path"),
305            ),
306            (
307                "az://container@account/path",
308                (ObjectStoreScheme::MicrosoftAzure, "path"),
309            ),
310            (
311                "abfs://container/path",
312                (ObjectStoreScheme::MicrosoftAzure, "path"),
313            ),
314            (
315                "abfs://container@account/path",
316                (ObjectStoreScheme::MicrosoftAzure, "path"),
317            ),
318            (
319                "abfss://container/path",
320                (ObjectStoreScheme::MicrosoftAzure, "path"),
321            ),
322            (
323                "abfss://container@account/path",
324                (ObjectStoreScheme::MicrosoftAzure, "path"),
325            ),
326            (
327                "adl://container/path",
328                (ObjectStoreScheme::MicrosoftAzure, "path"),
329            ),
330            (
331                "adl://container@account/path",
332                (ObjectStoreScheme::MicrosoftAzure, "path"),
333            ),
334            (
335                "gs://bucket/path",
336                (ObjectStoreScheme::GoogleCloudStorage, "path"),
337            ),
338            (
339                "gs://test.example.com/path",
340                (ObjectStoreScheme::GoogleCloudStorage, "path"),
341            ),
342            ("http://mydomain/path", (ObjectStoreScheme::Http, "path")),
343            ("https://mydomain/path", (ObjectStoreScheme::Http, "path")),
344            (
345                "s3://bucket/foo%20bar",
346                (ObjectStoreScheme::AmazonS3, "foo bar"),
347            ),
348            (
349                "s3://bucket/foo bar",
350                (ObjectStoreScheme::AmazonS3, "foo bar"),
351            ),
352            ("s3://bucket/😀", (ObjectStoreScheme::AmazonS3, "😀")),
353            (
354                "s3://bucket/%F0%9F%98%80",
355                (ObjectStoreScheme::AmazonS3, "😀"),
356            ),
357            (
358                "https://foo/bar%20baz",
359                (ObjectStoreScheme::Http, "bar baz"),
360            ),
361            (
362                "file:///bar%252Efoo",
363                (ObjectStoreScheme::Local, "bar%2Efoo"),
364            ),
365            (
366                "abfss://file_system@account.dfs.fabric.microsoft.com/",
367                (ObjectStoreScheme::MicrosoftAzure, ""),
368            ),
369            (
370                "abfss://file_system@account.dfs.fabric.microsoft.com/",
371                (ObjectStoreScheme::MicrosoftAzure, ""),
372            ),
373            (
374                "https://account.dfs.fabric.microsoft.com/",
375                (ObjectStoreScheme::MicrosoftAzure, ""),
376            ),
377            (
378                "https://account.dfs.fabric.microsoft.com/container",
379                (ObjectStoreScheme::MicrosoftAzure, ""),
380            ),
381            (
382                "https://account.dfs.fabric.microsoft.com/container/path",
383                (ObjectStoreScheme::MicrosoftAzure, "path"),
384            ),
385            (
386                "https://account.blob.fabric.microsoft.com/",
387                (ObjectStoreScheme::MicrosoftAzure, ""),
388            ),
389            (
390                "https://account.blob.fabric.microsoft.com/container",
391                (ObjectStoreScheme::MicrosoftAzure, ""),
392            ),
393            (
394                "https://account.blob.fabric.microsoft.com/container/path",
395                (ObjectStoreScheme::MicrosoftAzure, "path"),
396            ),
397        ];
398
399        for (s, (expected_scheme, expected_path)) in cases {
400            let url = Url::parse(s).unwrap();
401            let (scheme, path) = ObjectStoreScheme::parse(&url).unwrap();
402
403            assert_eq!(scheme, expected_scheme, "{s}");
404            assert_eq!(path, Path::parse(expected_path).unwrap(), "{s}");
405        }
406
407        let neg_cases = [
408            "unix:/run/foo.socket",
409            "file://remote/path",
410            "memory://remote/",
411        ];
412        for s in neg_cases {
413            let url = Url::parse(s).unwrap();
414            assert!(ObjectStoreScheme::parse(&url).is_err());
415        }
416    }
417
418    #[test]
419    fn test_url_spaces() {
420        let url = Url::parse("file:///my file with spaces").unwrap();
421        assert_eq!(url.path(), "/my%20file%20with%20spaces");
422        let (_, path) = parse_url(&url).unwrap();
423        assert_eq!(path.as_ref(), "my file with spaces");
424    }
425
426    #[test]
427    #[cfg(feature = "gcp")]
428    fn test_url_gcs_bearer_token_opts() {
429        let url = Url::parse("gs://bucket/path").unwrap();
430
431        for alias in ["google_bearer_token", "bearer_token"] {
432            let opts = [
433                (alias, "test-token"),
434                ("google_proxy_url", "https://example.com"),
435            ];
436
437            let (store, path) = parse_url_opts(&url, opts).unwrap();
438            assert_eq!(path.as_ref(), "path");
439            assert_eq!(store.to_string(), "GoogleCloudStorage(bucket)");
440        }
441    }
442
443    #[tokio::test]
444    #[cfg(all(
445        feature = "reqwest",
446        feature = "http-base",
447        not(target_arch = "wasm32")
448    ))]
449    async fn test_url_http() {
450        use crate::{ObjectStoreExt, client::mock_server::MockServer};
451        use http::{Response, header::USER_AGENT};
452
453        let server = MockServer::new().await;
454
455        server.push_fn(|r| {
456            assert_eq!(r.uri().path(), "/foo/bar");
457            assert_eq!(r.headers().get(USER_AGENT).unwrap(), "test_url");
458            Response::new(String::from("result"))
459        });
460
461        let test = format!("{}/foo/bar", server.url());
462        let opts = [("USER_AGENT", "test_url"), ("allow_http", "true")];
463        let url = test.parse().unwrap();
464        let (store, path) = parse_url_opts(&url, opts).unwrap();
465        assert_eq!(path.as_ref(), "foo/bar");
466
467        let res = store.get(&path).await.unwrap();
468        let body = res.bytes().await.unwrap();
469        let body = str::from_utf8(&body).unwrap();
470        assert_eq!(body, "result");
471
472        server.shutdown().await;
473    }
474}
object_store/parse.rs

object_store/
parse.rs
