Local fs start end offset (#101)

* Add TestList with empty case.

* TestList: add "one" case.

* TestList: add "many" case.

* Support StartOffset.

* Support EndOffset

* Include localfs in StartOffset/EndOffset GoDoc.

* Move offset filtering inside of filepath.Walk closure.

Author: snargleplax

localfs/store.go

@@ -153,8 +153,13 @@ func (l *LocalStore) List(ctx context.Context, query cloudstorage.Query) (*cloud
 			mdkey := strings.Replace(obj, ".metadata", "", 1)
 			metadatas[mdkey] = md
 		} else {
-
 			oname := strings.TrimPrefix(obj, "/")
+
+			if (query.StartOffset != "" && oname < query.StartOffset) ||
+				(query.EndOffset != "" && oname >= query.EndOffset) {
+				return nil
+			}
+
 			objects[obj] = &object{
 				name:      oname,
 				updated:   f.ModTime(),

localfs/store_test.go

@@ -7,11 +7,11 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/stretchr/testify/require"
-
 	"github.com/lytics/cloudstorage"
 	"github.com/lytics/cloudstorage/localfs"
 	"github.com/lytics/cloudstorage/testutils"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestAll(t *testing.T) {
@@ -111,3 +111,99 @@ func TestGetDir(t *testing.T) {
 	err = store.Delete(context.Background(), "test/index.html")
 	require.NoError(t, err)
 }
+
+func TestList(t *testing.T) {
+	t.Parallel()
+
+	for name, tt := range map[string]struct {
+		objs        map[string]string
+		q           cloudstorage.Query
+		startOffset string
+		want        []string
+	}{
+		"empty": {
+			objs: nil,
+			want: nil,
+		},
+		"one": {
+			objs: map[string]string{
+				"nimi": "ijo",
+			},
+			want: []string{"nimi"},
+		},
+		"many": {
+			objs: map[string]string{
+				"wan":    "loje",
+				"tu":     "jelo",
+				"tu wan": "laso",
+			},
+			want: []string{"wan", "tu", "tu wan"},
+		},
+		"start-offset-inclusive": {
+			objs: map[string]string{
+				"a": "ijo",
+				"b": "ijo",
+				"c": "ijo",
+			},
+			q: cloudstorage.Query{
+				StartOffset: "b",
+			},
+			want: []string{"b", "c"},
+		},
+		"end-offset-exclusive": {
+			objs: map[string]string{
+				"a": "ijo",
+				"b": "ijo",
+				"c": "ijo",
+			},
+			q: cloudstorage.Query{
+				EndOffset: "b",
+			},
+			want: []string{"a"},
+		},
+		"start-and-end-offsets-together": {
+			objs: map[string]string{
+				"a": "ijo",
+				"b": "ijo",
+				"c": "ijo",
+			},
+			q: cloudstorage.Query{
+				StartOffset: "b",
+				EndOffset:   "c",
+			},
+			want: []string{"b"},
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			ctx := context.Background()
+
+			tmpDir, err := ioutil.TempDir("/tmp", "getdir")
+			require.NoError(t, err)
+			t.Cleanup(func() { assert.NoError(t, os.RemoveAll(tmpDir)) })
+
+			store, err := localfs.NewLocalStore(
+				"list",
+				filepath.Join(tmpDir, "mockcloud"),
+				filepath.Join(tmpDir, "localcache"),
+			)
+			require.NoError(t, err)
+
+			for k, v := range tt.objs {
+				w, err := store.NewWriterWithContext(ctx, k, nil)
+				require.NoError(t, err)
+				_, err = w.Write([]byte(v))
+				require.NoError(t, err)
+				err = w.Close()
+				require.NoError(t, err)
+			}
+
+			got, err := store.List(ctx, tt.q)
+			require.NoError(t, err)
+			var names []string
+			for _, o := range got.Objects {
+				names = append(names, o.Name())
+			}
+			assert.ElementsMatch(t, tt.want, names)
+		})
+	}
+}

query.go

@@ -13,8 +13,8 @@ type Filter func(objects Objects) Objects
 type Query struct {
 	Delimiter   string   // Delimiter is most likely "/"
 	Prefix      string   // prefix (directory) to search for or object name if one file
-	StartOffset string   // (gcs only) "bar/", Only list objects lexicographically >= "bar/"
-	EndOffset   string   // (gcs only) "foo/", Only list objects lexicographically < "foo/"
+	StartOffset string   // (gcs/localfs only) "bar/", Only list objects lexicographically >= "bar/"
+	EndOffset   string   // (gcs/localfs only) "foo/", Only list objects lexicographically < "foo/"
 	Marker      string   // Next Page Marker if provided is a start next page fetch bookmark.
 	ShowHidden  bool     // Show hidden files?
 	Filters     []Filter // Applied to the result sets to filter out Objects (i.e. remove objects by extension)