diff --git a/environment.yml b/environment.yml index a37bca5..814e7a4 100644 --- a/environment.yml +++ b/environment.yml @@ -5,7 +5,6 @@ dependencies: - python>=3.12, <=3.14 - pip - cloudpickle>=3.1.2 - - boto3 - tiledb>=2.29.2 - tiledb-py>=0.35.2 - libpdal-core>=2.9.2 diff --git a/src/silvimetric/__init__.py b/src/silvimetric/__init__.py index 1f45765..0a9d991 100644 --- a/src/silvimetric/__init__.py +++ b/src/silvimetric/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.0.0' +__version__ = '2.0.1' from .resources.bounds import Bounds from .resources.extents import Extents diff --git a/src/silvimetric/resources/data.py b/src/silvimetric/resources/data.py index 98a61d5..e9aba79 100644 --- a/src/silvimetric/resources/data.py +++ b/src/silvimetric/resources/data.py @@ -4,7 +4,7 @@ from urllib.parse import urlparse from typing import Optional -import boto3 +import tiledb import pdal import numpy as np @@ -106,17 +106,9 @@ def get_pipeline(self) -> pdal.Pipeline: # aren't, go make_pipeline using some options that # process the data if self.is_pipeline(): - if 's3://' in self.filename: - s3 = boto3.client('s3') - parsed = urlparse(self.filename) - bucket = parsed.netloc - key = parsed.path[1:] - res = s3.get_object(Bucket=bucket, Key=key) - j = res['Body'].read().decode('utf-8') - else: - p = pathlib.Path(self.filename) - j = p.read_bytes().decode('utf-8') - stages = pdal.pipeline._parse_stages(j) + vfs = tiledb.VFS() + pipeline_str = vfs.open(self.filename).read() + stages = pdal.pipeline._parse_stages(pipeline_str) pipeline = pdal.Pipeline(stages) else: pipeline = self.make_pipeline() @@ -220,17 +212,9 @@ def get_reader(self) -> pdal.Reader: """ if self.is_pipeline(): if self.pipeline is None: - if 's3://' in self.filename: - s3 = boto3.client('s3') - parsed = urlparse(self.filename) - bucket = parsed.netloc - key = parsed.path[1:] - res = s3.get_object(Bucket=bucket, Key=key) - j = res['Body'].read().decode('utf-8') - else: - p = pathlib.Path(self.filename) - j = p.read_bytes().decode('utf-8') - stages = pdal.pipeline._parse_stages(j) + vfs = tiledb.VFS() + pipeline_str = vfs.open(self.filename).read() + stages = pdal.pipeline._parse_stages(pipeline_str) else: stages = self.pipeline.stages diff --git a/tests/fixtures/shatter_fixtures.py b/tests/fixtures/shatter_fixtures.py index 927c58d..ddb73fd 100644 --- a/tests/fixtures/shatter_fixtures.py +++ b/tests/fixtures/shatter_fixtures.py @@ -3,6 +3,7 @@ from typing_extensions import Generator from uuid import uuid4 import os +import tiledb from silvimetric import __version__ as svversion from silvimetric import StorageConfig, ShatterConfig, Storage, Log, Bounds @@ -56,13 +57,12 @@ def s3_storage_config( @pytest.fixture(scope='function') def s3_storage( s3_storage_config: StorageConfig, + s3_uri: str ) -> Generator[Storage, None, None]: - import subprocess + vfs = tiledb.VFS() yield Storage.create(s3_storage_config) - subprocess.call( - ['aws', 's3', 'rm', '--recursive', s3_storage_config.tdb_dir] - ) + vfs.remove_dir(s3_uri) @pytest.fixture(scope='function')