Add the ability to read files from Starlark (#1045)

* Add the ability to read files from Starlark

Starlark scripts can now load arbitrary files from their bundle and read
them using a Python-like `open()` and `read()` API. A `readall()`
convenience function is also provided.

The examples have been updated to use this functionality where relevant.
For example:

```
load("icon.png", icon = "file")

BTC_ICON = icon.readall()
```

The `pixlet render` command can now accept an entire directory
containing multiple Starlark files and static resources.

* Remove `file.open` and `read()`

Remove the `open()` and `read()` functions, so the only way to read a
file is `readall()`.

If there is demand for the other methods, we can add them back later.

* Add directory support to `pixlet serve`

* Implement `pixlet bundle` for directories

* Fix path problems on Windows

`fs.FS` always uses slash-separated paths, even on Windows. So we should
be manipulating those paths with `path`, not `filepath`.

* Switch tarfs implementations for Windows support

It appears that `nlepage/go-tarfs` correctly supports Windows by [using
`path` instead of `filepath`][1].

[1]: nlepage/go-tarfs#7

.gitignore

@@ -2,8 +2,8 @@
 .DS_Store
 
 # Rendered Apps
-examples/*.webp
-examples/*.gif
+examples/**/*.webp
+examples/**/*.gif
 
 # Pixlet Binary
 pixlet

README.md

@@ -60,7 +60,7 @@ def main():
 Render and serve it with:
 
 ```console
-curl https://raw.githubusercontent.com/tidbyt/pixlet/main/examples/hello_world.star | \
+curl https://raw.githubusercontent.com/tidbyt/pixlet/main/examples/hello_world/hello_world.star | \
   pixlet serve /dev/stdin
 ```
 
@@ -135,7 +135,7 @@ to show the Bitcoin tracker on your Tidbyt:
 
 ```console
 # render the bitcoin example
-pixlet render examples/bitcoin.star
+pixlet render examples/bitcoin/bitcoin.star
 
 # login to your Tidbyt account
 pixlet login
@@ -144,7 +144,7 @@ pixlet login
 pixlet devices
 
 # push to your favorite Tidbyt
-pixlet push <YOUR DEVICE ID> examples/bitcoin.webp
+pixlet push <YOUR DEVICE ID> examples/bitcoin/bitcoin.webp
 ```
 
 To get the ID for a device, run `pixlet devices`. Alternatively, you can
@@ -158,8 +158,8 @@ If all goes well, you should see the Bitcoin tracker appear on your Tidbyt:
 Pushing an applet to your Tidbyt without an installation ID simply displays your applet one time. If you would like your applet to continously display as part of the rotation, add an installation ID to the push command:
 
 ```console
-pixlet render examples/bitcoin.star
-pixlet push --installation-id <INSTALLATION ID> <YOUR DEVICE ID> examples/bitcoin.webp
+pixlet render examples/bitcoin/bitcoin.star
+pixlet push --installation-id <INSTALLATION ID> <YOUR DEVICE ID> examples/bitcoin/bitcoin.webp
 ```
 
 For example, if we set the `installationID` to "Bitcoin", it would appear in the mobile app as follows:

bundle/bundle.go

@@ -7,10 +7,14 @@ import (
 	"compress/gzip"
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
 
+	"github.com/nlepage/go-tarfs"
+
 	"tidbyt.dev/pixlet/manifest"
+	"tidbyt.dev/pixlet/runtime"
 )
 
 const (
@@ -26,16 +30,12 @@ const (
 
 // AppBundle represents the unpacked bundle in our system.
 type AppBundle struct {
-	Source   []byte
 	Manifest *manifest.Manifest
+	Source   fs.FS
 }
 
-// InitFromPath translates a directory containing an app manifest and source
-// into an AppBundle.
-func InitFromPath(dir string) (*AppBundle, error) {
-	// Load manifest
-	path := filepath.Join(dir, manifest.ManifestFileName)
-	m, err := os.Open(path)
+func fromFS(fs fs.FS) (*AppBundle, error) {
+	m, err := fs.Open(manifest.ManifestFileName)
 	if err != nil {
 		return nil, fmt.Errorf("could not open manifest: %w", err)
 	}
@@ -46,80 +46,39 @@ func InitFromPath(dir string) (*AppBundle, error) {
 		return nil, fmt.Errorf("could not load manifest: %w", err)
 	}
 
-	// Load source
-	path = filepath.Join(dir, man.FileName)
-	s, err := os.Open(path)
-	if err != nil {
-		return nil, fmt.Errorf("could not open app source: %w", err)
-	}
-	defer s.Close()
-
-	src, err := io.ReadAll(s)
-	if err != nil {
-		return nil, fmt.Errorf("could not read app source: %w", err)
-	}
-
 	// Create app bundle struct
 	return &AppBundle{
 		Manifest: man,
-		Source:   src,
+		Source:   fs,
 	}, nil
 }
 
+// InitFromPath translates a directory containing an app manifest and source
+// into an AppBundle.
+func InitFromPath(dir string) (*AppBundle, error) {
+	return fromFS(os.DirFS(dir))
+}
+
 // LoadBundle loads a compressed archive into an AppBundle.
 func LoadBundle(in io.Reader) (*AppBundle, error) {
 	gzr, err := gzip.NewReader(in)
 	if err != nil {
-		return nil, fmt.Errorf("could not create gzip reader: %w", err)
+		return nil, fmt.Errorf("creating gzip reader: %w", err)
 	}
 	defer gzr.Close()
 
-	tr := tar.NewReader(gzr)
-	ab := &AppBundle{}
-
-	for {
-		header, err := tr.Next()
-
-		switch {
-		case err == io.EOF:
-			// If there are no more files in the bundle, validate and return it.
-			if ab.Manifest == nil {
-				return nil, fmt.Errorf("could not find manifest in archive")
-			}
-			if ab.Source == nil {
-				return nil, fmt.Errorf("could not find source in archive")
-			}
-			return ab, nil
-		case err != nil:
-			// If there is an error, return immediately.
-			return nil, fmt.Errorf("could not read archive: %w", err)
-		case header == nil:
-			// If for some reason we end up with a blank header, continue to the
-			// next one.
-			continue
-		case header.Name == AppSourceName:
-			// Load the app source.
-			buff := make([]byte, header.Size)
-			_, err := io.ReadFull(tr, buff)
-			if err != nil {
-				return nil, fmt.Errorf("could not read source from archive: %w", err)
-			}
-			ab.Source = buff
-		case header.Name == manifest.ManifestFileName:
-			// Load the app manifest.
-			buff := make([]byte, header.Size)
-			_, err := io.ReadFull(tr, buff)
-			if err != nil {
-				return nil, fmt.Errorf("could not read manifest from archive: %w", err)
-			}
+	// read the entire tarball into memory so that we can seek
+	// around it, and so that the underlying reader can be closed.
+	var b bytes.Buffer
+	io.Copy(&b, gzr)
 
-			man, err := manifest.LoadManifest(bytes.NewReader(buff))
-			if err != nil {
-				return nil, fmt.Errorf("could not load manifest: %w", err)
-			}
-			ab.Manifest = man
-		}
+	r := bytes.NewReader(b.Bytes())
+	fs, err := tarfs.New(r)
+	if err != nil {
+		return nil, fmt.Errorf("creating tarfs: %w", err)
 	}
+
+	return fromFS(fs)
 }
 
 // WriteBundleToPath is a helper to be able to write the bundle to a provided
@@ -137,6 +96,16 @@ func (b *AppBundle) WriteBundleToPath(dir string) error {
 
 // WriteBundle writes a compressed archive to the provided writer.
 func (ab *AppBundle) WriteBundle(out io.Writer) error {
+	// we don't want to naively write the entire source FS to the tarball,
+	// since it could contain a lot of extraneous files. instead, run the
+	// applet and interrogate it for the files it needs to include in the
+	// bundle.
+	app, err := runtime.NewAppletFromFS(ab.Manifest.ID, ab.Source, runtime.WithPrintDisabled())
+	if err != nil {
+		return fmt.Errorf("loading applet for bundling: %w", err)
+	}
+	bundleFiles := app.PathsForBundle()
+
 	// Setup writers.
 	gzw := gzip.NewWriter(out)
 	defer gzw.Close()
@@ -146,7 +115,7 @@ func (ab *AppBundle) WriteBundle(out io.Writer) error {
 
 	// Write manifest.
 	buff := &bytes.Buffer{}
-	err := ab.Manifest.WriteManifest(buff)
+	err = ab.Manifest.WriteManifest(buff)
 	if err != nil {
 		return fmt.Errorf("could not write manifest to buffer: %w", err)
 	}
@@ -166,19 +135,37 @@ func (ab *AppBundle) WriteBundle(out io.Writer) error {
 		return fmt.Errorf("could not write manifest to archive: %w", err)
 	}
 
-	// Write source.
-	hdr = &tar.Header{
-		Name: AppSourceName,
-		Mode: 0600,
-		Size: int64(len(ab.Source)),
-	}
-	err = tw.WriteHeader(hdr)
-	if err != nil {
-		return fmt.Errorf("could not write source header: %w", err)
-	}
-	_, err = tw.Write(ab.Source)
-	if err != nil {
-		return fmt.Errorf("could not write source to archive: %w", err)
+	// write sources.
+	for _, path := range bundleFiles {
+		stat, err := fs.Stat(ab.Source, path)
+		if err != nil {
+			return fmt.Errorf("could not stat %s: %w", path, err)
+		}
+
+		hdr, err := tar.FileInfoHeader(stat, "")
+		if err != nil {
+			return fmt.Errorf("creating header for %s: %w", path, err)
+		}
+		hdr.Name = filepath.ToSlash(path)
+
+		err = tw.WriteHeader(hdr)
+		if err != nil {
+			return fmt.Errorf("writing header for %s: %w", path, err)
+		}
+
+		if !stat.IsDir() {
+			file, err := ab.Source.Open(path)
+			if err != nil {
+				return fmt.Errorf("opening file %s: %w", path, err)
+			}
+
+			written, err := io.Copy(tw, file)
+			if err != nil {
+				return fmt.Errorf("writing file %s: %w", path, err)
+			} else if written != stat.Size() {
+				return fmt.Errorf("did not write entire file %s: %w", path, err)
+			}
+		}
 	}
 
 	return nil

bundle/bundle_test.go

@@ -14,7 +14,7 @@ func TestBundleWriteAndLoad(t *testing.T) {
 	ab, err := bundle.InitFromPath("testdata/testapp")
 	assert.NoError(t, err)
 	assert.Equal(t, "test-app", ab.Manifest.ID)
-	assert.True(t, len(ab.Source) > 0)
+	assert.NotNil(t, ab.Source)
 
 	// Create a temp directory.
 	dir, err := os.MkdirTemp("", "")
@@ -32,7 +32,23 @@ func TestBundleWriteAndLoad(t *testing.T) {
 	newBun, err := bundle.LoadBundle(f)
 	assert.NoError(t, err)
 	assert.Equal(t, "test-app", newBun.Manifest.ID)
-	assert.True(t, len(ab.Source) > 0)
+	assert.NotNil(t, ab.Source)
+
+	// Ensure the loaded bundle contains the files we expect.
+	filesExpected := []string{
+		"manifest.yaml",
+		"test_app.star",
+		"test.txt",
+		"a_subdirectory/hi.jpg",
+	}
+	for _, file := range filesExpected {
+		_, err := newBun.Source.Open(file)
+		assert.NoError(t, err)
+	}
+
+	// Ensure the loaded bundle does not contain any extra files.
+	_, err = newBun.Source.Open("unused.txt")
+	assert.ErrorIs(t, err, os.ErrNotExist)
 }
 func TestLoadBundle(t *testing.T) {
 	f, err := os.Open("testdata/bundle.tar.gz")
@@ -41,7 +57,7 @@ func TestLoadBundle(t *testing.T) {
 	ab, err := bundle.LoadBundle(f)
 	assert.NoError(t, err)
 	assert.Equal(t, "test-app", ab.Manifest.ID)
-	assert.True(t, len(ab.Source) > 0)
+	assert.NotNil(t, ab.Source)
 }
 func TestLoadBundleExcessData(t *testing.T) {
 	f, err := os.Open("testdata/excess-files.tar.gz")
@@ -51,5 +67,5 @@ func TestLoadBundleExcessData(t *testing.T) {
 	ab, err := bundle.LoadBundle(f)
 	assert.NoError(t, err)
 	assert.Equal(t, "test-app", ab.Manifest.ID)
-	assert.True(t, len(ab.Source) > 0)
+	assert.NotNil(t, ab.Source)
 }

bundle/testdata/testapp/manifest.yaml

@@ -4,5 +4,3 @@ name: Test App
 summary: For Testing
 desc: It's an app for testing.
 author: Test Dev
-fileName: test_app.star
-packageName: testapp

bundle/testdata/testapp/test_app.star

@@ -5,11 +5,17 @@ Description: It's an app for testing.
 Author: Test Dev
 """
 
+load("a_subdirectory/hi.jpg", hi_jpeg = "file")
 load("render.star", "render")
 load("schema.star", "schema")
+load("test.txt", test_txt = "file")
 
 DEFAULT_WHO = "world"
 
+TEST_TXT_CONTENT = test_txt.readall()
+
+HI_JPEG_BYTES = hi_jpeg.readall("rb")
+
 def main(config):
     who = config.str("who", DEFAULT_WHO)
     message = "Hello, {}!".format(who)

bundle/testdata/testapp/unused.txt

@@ -0,0 +1 @@
+this file is not used in the app

cmd/community/manifestprompt.go

@@ -51,12 +51,10 @@ func ManifestPrompt() (*manifest.Manifest, error) {
 	}
 
 	return &manifest.Manifest{
-		ID:          manifest.GenerateID(name),
-		Name:        name,
-		Summary:     summary,
-		Desc:        desc,
-		Author:      author,
-		FileName:    manifest.GenerateFileName(name),
-		PackageName: manifest.GeneratePackageName(name),
+		ID:      manifest.GenerateID(name),
+		Name:    name,
+		Summary: summary,
+		Desc:    desc,
+		Author:  author,
 	}, nil
 }

cmd/community/validatemanifest.go

@@ -12,7 +12,6 @@ import (
 var ValidateManifestAppFileName string
 
 func init() {
-	ValidateManifestCmd.Flags().StringVarP(&ValidateManifestAppFileName, "app-file-name", "a", "", "ensures the app file name is the same as the manifest")
 }
 
 var ValidateManifestCmd = &cobra.Command{
@@ -47,9 +46,5 @@ func ValidateManifest(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("couldn't validate manifest: %w", err)
 	}
 
-	if ValidateManifestAppFileName != "" && m.FileName != ValidateManifestAppFileName {
-		return fmt.Errorf("app name doesn't match: %s != %s", ValidateManifestAppFileName, m.FileName)
-	}
-
 	return nil
 }