From b13db25f39cc00b2486bb820dbb8945a7f5497bd Mon Sep 17 00:00:00 2001 From: Ansuman Sahoo Date: Thu, 26 Feb 2026 15:05:29 +0530 Subject: [PATCH] Ensure only RAW images in cache while keeping orginal image Signed-off-by: Ansuman Sahoo --- pkg/cacheutil/cacheutil.go | 2 +- pkg/downloader/downloader.go | 211 ++++++++++++++++-- pkg/downloader/downloader_test.go | 134 +++++++++++ pkg/driver/driver.go | 15 +- .../krunkit/krunkit_driver_darwin_arm64.go | 7 +- pkg/driver/qemu/qemu.go | 2 +- pkg/driver/qemu/qemu_driver.go | 11 +- pkg/driver/vz/vz_driver_darwin.go | 11 +- pkg/driver/wsl2/fs.go | 2 +- pkg/driverutil/disk.go | 22 +- pkg/fileutils/download.go | 11 +- pkg/imgutil/nativeimgutil/nativeimgutil.go | 16 ++ pkg/instance/start.go | 10 +- website/content/en/docs/dev/internals.md | 3 + 14 files changed, 413 insertions(+), 44 deletions(-) diff --git a/pkg/cacheutil/cacheutil.go b/pkg/cacheutil/cacheutil.go index be66f800..0771b526 100644 --- a/pkg/cacheutil/cacheutil.go +++ b/pkg/cacheutil/cacheutil.go @@ -43,7 +43,7 @@ func EnsureNerdctlArchiveCache(ctx context.Context, y *limatype.LimaYAML, create return path, nil } } - path, err := fileutils.DownloadFile(ctx, "", f, false, "the nerdctl archive", *y.Arch) + path, err := fileutils.DownloadFile(ctx, "", f, false, "the nerdctl archive", *y.Arch, nil) if err != nil { errs[i] = err continue diff --git a/pkg/downloader/downloader.go b/pkg/downloader/downloader.go index 9c8ecaf0..a9e06e43 100644 --- a/pkg/downloader/downloader.go +++ b/pkg/downloader/downloader.go @@ -16,16 +16,21 @@ import ( "os/exec" "path" "path/filepath" + "slices" "strings" "sync/atomic" "time" "github.com/cheggaaa/pb/v3" "github.com/containerd/continuity/fs" + "github.com/lima-vm/go-qcow2reader/image/raw" "github.com/opencontainers/go-digest" "github.com/sirupsen/logrus" "github.com/lima-vm/lima/v2/pkg/httpclientutil" + "github.com/lima-vm/lima/v2/pkg/imgutil/nativeimgutil" + "github.com/lima-vm/lima/v2/pkg/imgutil/proxyimgutil" + "github.com/lima-vm/lima/v2/pkg/iso9660util" "github.com/lima-vm/lima/v2/pkg/localpathutil" "github.com/lima-vm/lima/v2/pkg/lockutil" "github.com/lima-vm/lima/v2/pkg/progressbar" @@ -57,10 +62,11 @@ type Result struct { } type options struct { - cacheDir string // default: empty (disables caching) - decompress bool // default: false (keep compression) - description string // default: url - expectedDigest digest.Digest + cacheDir string // default: empty (disables caching) + decompress bool // default: false (keep compression) + description string // default: url + expectedDigest digest.Digest + supportedImageFormats []string } func (o *options) apply(opts []Opt) error { @@ -111,6 +117,13 @@ func WithDecompress(decompress bool) Opt { } } +func WithImageFormats(supportedImageFormats []string) Opt { + return func(o *options) error { + o.supportedImageFormats = supportedImageFormats + return nil + } +} + // WithExpectedDigest is used to validate the downloaded file against the expected digest. // // The digest is not verified in the following cases: @@ -267,6 +280,81 @@ func getCached(ctx context.Context, localPath, remote string, o options) (*Resul if _, err := os.Stat(shadData); err != nil { return nil, nil } + if _, err := os.Stat(shadDigest); err == nil { + if err := validateCachedDigest(shadDigest, o.expectedDigest); err != nil { + return nil, err + } + } else if match, lmCached, lmRemote, err := matchLastModified(ctx, shadTime, remote); err != nil { + logrus.WithError(err).Info("Failed to retrieve last-modified for cached digest-less image; using cached image.") + } else if !match { + logrus.Infof("Re-downloading digest-less image: last-modified mismatch (cached: %#q, remote: %#q)", lmCached, lmRemote) + return nil, nil + } + + // Some drivers (e.g. vz) can only boot raw images. When the driver tells us + // which formats it supports and the cached image is not one of them, we save + // a converted raw copy next to the original at /imgconv/raw. See + // website/content/en/docs/dev/internals.md for the layout. + if len(o.supportedImageFormats) > 0 { + isISO, err := iso9660util.IsISO9660(shadData) + if err != nil { + logrus.WithError(err).Debugf("Skipping cache image conversion for %q (unable to check ISO9660)", shadData) + } else if !isISO { + imageFormat, err := nativeimgutil.DetectFormat(shadData) + if err != nil { + logrus.WithError(err).Debugf("Skipping cache image conversion for %q (unable to detect format)", shadData) + } else if !slices.Contains(o.supportedImageFormats, imageFormat) { + rawImgConvPath := filepath.Join(shad, "imgconv", "raw") + rawImgConvDigestPath := filepath.Join(shad, "imgconv", "raw.digest") + + needConvert := false + if rawStat, err := os.Stat(rawImgConvPath); err != nil { + if errors.Is(err, os.ErrNotExist) { + needConvert = true + } else { + return nil, err + } + } else { + origStat, err := os.Stat(shadData) + if err != nil { + return nil, err + } + if origStat.ModTime().After(rawStat.ModTime()) { + needConvert = true + } + } + + if needConvert { + logrus.Infof("Converted raw image is missing or stale; (re)converting now.") + converted, rawDigest, err := ensureRawInCache(ctx, shadData, imageFormat, o.expectedDigest) + if err != nil { + return nil, err + } + shadData = converted + if o.expectedDigest != "" { + o.expectedDigest = rawDigest + shadDigest = rawImgConvDigestPath + } + } else { + shadData = rawImgConvPath + if o.expectedDigest != "" { + if currentDigestData, err := os.ReadFile(rawImgConvDigestPath); err == nil { + currentDigest := strings.TrimSpace(string(currentDigestData)) + if d, err := digest.Parse(currentDigest); err == nil { + o.expectedDigest = d + shadDigest = rawImgConvDigestPath + } else { + return nil, fmt.Errorf("invalid digest in raw digest file %q: %w", rawImgConvDigestPath, err) + } + } else { + return nil, fmt.Errorf("failed to read raw digest file %q: %w", rawImgConvDigestPath, err) + } + } + } + } + } + } + ext := path.Ext(remote) logrus.Debugf("file %#q is cached as %#q", localPath, shadData) if _, err := os.Stat(shadDigest); err == nil { @@ -279,15 +367,8 @@ func getCached(ctx context.Context, localPath, remote string, o options) (*Resul return nil, err } } else { - if match, lmCached, lmRemote, err := matchLastModified(ctx, shadTime, remote); err != nil { - logrus.WithError(err).Info("Failed to retrieve last-modified for cached digest-less image; using cached image.") - } else if match { - if err := copyLocal(ctx, localPath, shadData, ext, o.decompress, o.description, o.expectedDigest); err != nil { - return nil, err - } - } else { - logrus.Infof("Re-downloading digest-less image: last-modified mismatch (cached: %#q, remote: %#q)", lmCached, lmRemote) - return nil, nil + if err := copyLocal(ctx, localPath, shadData, ext, o.decompress, o.description, o.expectedDigest); err != nil { + return nil, err } } res := &Result{ @@ -323,6 +404,34 @@ func fetch(ctx context.Context, localPath, remote string, o options) (*Result, e return nil, err } } + + // If the driver cannot use the format we just downloaded, save a converted + // raw copy at /imgconv/raw, keeping the original /data as-is. + // + // Gated on WithImageFormats() (the caller's signal that this is a VM disk + // image) and on the file not being an ISO 9660 image. + if len(o.supportedImageFormats) > 0 { + isISO, err := iso9660util.IsISO9660(shadData) + if err != nil { + logrus.WithError(err).Debugf("Skipping cache image conversion for %q (unable to check ISO9660)", shadData) + } else if !isISO { + format, err := nativeimgutil.DetectFormat(shadData) + if err != nil { + logrus.WithError(err).Debugf("Skipping cache image conversion for %q (unable to detect format)", shadData) + } else if !slices.Contains(o.supportedImageFormats, format) { + converted, rawDigest, err := ensureRawInCache(ctx, shadData, format, o.expectedDigest) + if err != nil { + return nil, fmt.Errorf("failed to convert image to raw: %w", err) + } else if converted != "" { + shadData = converted + if o.expectedDigest != "" { + o.expectedDigest = rawDigest + } + } + } + } + } + // no need to pass the digest to copyLocal(), as we already verified the digest if err := copyLocal(ctx, localPath, shadData, ext, o.decompress, "", ""); err != nil { return nil, err @@ -337,6 +446,82 @@ func fetch(ctx context.Context, localPath, remote string, o options) (*Result, e return res, nil } +// ensureRawInCache converts any image to raw and places it in the cache(imgconv/raw). It also creates a +// digest file for the raw image(imgconv/raw.digest). Returns the converted image path, the raw digest, and any error. +func ensureRawInCache(ctx context.Context, imagePath, format string, originalDigest digest.Digest) (string, digest.Digest, error) { + imgConvPath := filepath.Join(filepath.Dir(imagePath), "imgconv") + if err := os.MkdirAll(imgConvPath, 0o700); err != nil { + return "", "", err + } + rawImgConvPath := filepath.Join(imgConvPath, "raw") + + logrus.Infof("Converting %s image to raw sparse format in cache: %q", format, rawImgConvPath) + rawPathTmp := filepath.Join(imgConvPath, "raw.tmp") + defer os.Remove(rawPathTmp) + diskUtil := proxyimgutil.NewDiskUtil(ctx) + if err := diskUtil.Convert(ctx, raw.Type, imagePath, rawPathTmp, nil, false); err != nil { + return "", "", fmt.Errorf("failed to convert %q to raw: %w", imagePath, err) + } + + // Ensure the image is sparse to save cache space. + rawTmpF, err := os.OpenFile(rawPathTmp, os.O_RDWR, 0o644) + if err != nil { + return "", "", fmt.Errorf("failed to open raw tmp file %q: %w", rawPathTmp, err) + } + fi, err := rawTmpF.Stat() + if err != nil { + _ = rawTmpF.Close() + return "", "", fmt.Errorf("failed to stat raw tmp file %q: %w", rawPathTmp, err) + } + if err := diskUtil.MakeSparse(ctx, rawTmpF, fi.Size()); err != nil { + logrus.WithError(err).Warnf("Failed to make %q sparse (non-fatal)", rawPathTmp) + } + if err := rawTmpF.Close(); err != nil { + return "", "", fmt.Errorf("failed to close raw tmp file %q: %w", rawPathTmp, err) + } + + if err := os.Remove(rawImgConvPath); err != nil && !errors.Is(err, os.ErrNotExist) { + return "", "", fmt.Errorf("failed to remove stale raw image %q: %w", rawImgConvPath, err) + } + if err := os.Rename(rawPathTmp, rawImgConvPath); err != nil { + return "", "", fmt.Errorf("failed to replace original with raw image: %w", err) + } + + algo := digest.Canonical + if originalDigest != "" { + algo = originalDigest.Algorithm() + } + rawDigest, err := calculateFileDigest(rawImgConvPath, algo) + if err != nil { + return "", "", fmt.Errorf("failed to calculate digest of raw image: %w", err) + } + + rawDigestPath := filepath.Join(imgConvPath, "raw.digest") + rawDigestPathTmp := rawDigestPath + ".tmp" + defer os.Remove(rawDigestPathTmp) + if err := os.WriteFile(rawDigestPathTmp, []byte(rawDigest.String()), 0o644); err != nil { + return "", "", fmt.Errorf("failed to write raw digest file: %w", err) + } + if err := os.Remove(rawDigestPath); err != nil && !errors.Is(err, os.ErrNotExist) { + return "", "", fmt.Errorf("failed to remove stale raw digest file %q: %w", rawDigestPath, err) + } + if err := os.Rename(rawDigestPathTmp, rawDigestPath); err != nil { + return "", "", fmt.Errorf("failed to rename raw digest file: %w", err) + } + + return rawImgConvPath, rawDigest, nil +} + +func calculateFileDigest(path string, algo digest.Algorithm) (digest.Digest, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + + return algo.FromReader(f) +} + // Cached checks if the remote resource is in the cache. // // Download caches the remote resource if WithCache or WithCacheDir option is specified. diff --git a/pkg/downloader/downloader_test.go b/pkg/downloader/downloader_test.go index 905f6987..177b6d90 100644 --- a/pkg/downloader/downloader_test.go +++ b/pkg/downloader/downloader_test.go @@ -398,3 +398,137 @@ func TestDownloadCompressed(t *testing.T) { assert.Equal(t, string(got), string(testDownloadCompressedContents)) }) } + +// This test simulates the end-to-end flow of downloading a remote image and converting it. +func TestDownloadImageConversion(t *testing.T) { + _, err := exec.LookPath("qemu-img") + if err != nil { + t.Skipf("qemu-img does not seem installed: %v", err) + } + + remoteDir := t.TempDir() + ts := httptest.NewServer(http.FileServer(http.Dir(remoteDir))) + t.Cleanup(ts.Close) + + qcow2Path := filepath.Join(remoteDir, "test.qcow2") + assert.NilError(t, exec.CommandContext(t.Context(), "qemu-img", "create", "-f", "qcow2", qcow2Path, "64K").Run()) + + t.Run("without-digest", func(t *testing.T) { + ctx := t.Context() + cacheDir := t.TempDir() + localPath := filepath.Join(t.TempDir(), "local") + remoteURL := ts.URL + "/test.qcow2" + + // 1. First download, should convert to raw + r, err := Download(ctx, localPath, remoteURL, + WithCacheDir(cacheDir), + WithDescription("image"), + WithDecompress(true), + WithImageFormats([]string{"raw"}), + ) + assert.NilError(t, err) + assert.Equal(t, StatusDownloaded, r.Status) + + shad := cacheDirectoryPath(cacheDir, remoteURL) + rawPath := filepath.Join(shad, "imgconv", "raw") + _, err = os.Stat(rawPath) + assert.NilError(t, err) + + // 2. Second download, should use cached raw + localPath2 := filepath.Join(t.TempDir(), "local2") + r, err = Download(ctx, localPath2, remoteURL, + WithCacheDir(cacheDir), + WithDescription("image"), + WithDecompress(true), + WithImageFormats([]string{"raw"}), + ) + assert.NilError(t, err) + assert.Equal(t, StatusUsedCache, r.Status) + assert.Equal(t, rawPath, r.CachePath) + }) + + t.Run("with-digest", func(t *testing.T) { + ctx := t.Context() + cacheDir := t.TempDir() + localPath := filepath.Join(t.TempDir(), "local") + remoteURL := ts.URL + "/test.qcow2" + + content, err := os.ReadFile(qcow2Path) + assert.NilError(t, err) + originalDigest := digest.SHA256.FromBytes(content) + + // 1. First download, should convert to raw + r, err := Download(ctx, localPath, remoteURL, + WithCacheDir(cacheDir), + WithDescription("image"), + WithDecompress(true), + WithImageFormats([]string{"raw"}), + WithExpectedDigest(originalDigest), + ) + assert.NilError(t, err) + assert.Equal(t, StatusDownloaded, r.Status) + + shad := cacheDirectoryPath(cacheDir, remoteURL) + rawPath := filepath.Join(shad, "imgconv", "raw") + _, err = os.Stat(rawPath) + assert.NilError(t, err) + + // 2. Second download, should use cached raw + localPath2 := filepath.Join(t.TempDir(), "local2") + r, err = Download(ctx, localPath2, remoteURL, + WithCacheDir(cacheDir), + WithDescription("image"), + WithDecompress(true), + WithImageFormats([]string{"raw"}), + WithExpectedDigest(originalDigest), + ) + assert.NilError(t, err) + assert.Equal(t, StatusUsedCache, r.Status) + assert.Equal(t, rawPath, r.CachePath) + }) +} + +// This test focuses specifically on the scenario where the source image +// is already in the Lima cache but the converted version does not exist yet. +func TestDownloadImageConversionCached(t *testing.T) { + _, err := exec.LookPath("qemu-img") + if err != nil { + t.Skipf("qemu-img does not seem installed: %v", err) + } + + ctx := t.Context() + tmpDir := t.TempDir() + cacheDir := filepath.Join(tmpDir, "cache") + remoteURL := "https://example.com/test.qcow2" + + // Pre-populate cache with a qcow2 file + shad := cacheDirectoryPath(cacheDir, remoteURL) + assert.NilError(t, os.MkdirAll(shad, 0o700)) + shadData := filepath.Join(shad, "data") + assert.NilError(t, exec.CommandContext(t.Context(), "qemu-img", "create", "-f", "qcow2", shadData, "64K").Run()) + + // Provide a cached digest to avoid HTTP HEAD / re-download. + content, err := os.ReadFile(shadData) + assert.NilError(t, err) + originalDigest := digest.SHA256.FromBytes(content) + shadDigest := filepath.Join(shad, "sha256.digest") + assert.NilError(t, os.WriteFile(shadDigest, []byte(originalDigest.String()), 0o644)) + + localPath := filepath.Join(tmpDir, "local") + + // Call Download, it should find qcow2 in cache, see it's not supported, and convert it + r, err := Download(ctx, localPath, remoteURL, + WithCacheDir(cacheDir), + WithDescription("image"), + WithImageFormats([]string{"raw"}), + WithExpectedDigest(originalDigest), + ) + assert.NilError(t, err) + assert.Equal(t, StatusUsedCache, r.Status) + + // Verify conversion happened + rawPath := filepath.Join(shad, "imgconv", "raw") + _, err = os.Stat(rawPath) + assert.NilError(t, err) + assert.Equal(t, rawPath, r.CachePath) +} diff --git a/pkg/driver/driver.go b/pkg/driver/driver.go index b500eb2a..ee15816f 100644 --- a/pkg/driver/driver.go +++ b/pkg/driver/driver.go @@ -129,11 +129,12 @@ type Info struct { } type DriverFeatures struct { - CanRunGUI bool `json:"canRunGui,omitempty"` - DynamicSSHAddress bool `json:"dynamicSSHAddress"` - StaticSSHPort bool `json:"staticSSHPort"` - SkipSocketForwarding bool `json:"skipSocketForwarding"` - NoCloudInit bool `json:"noCloudInit"` - RosettaEnabled bool `json:"rosettaEnabled"` - RosettaBinFmt bool `json:"rosettaBinFmt"` + CanRunGUI bool `json:"canRunGui,omitempty"` + DynamicSSHAddress bool `json:"dynamicSSHAddress"` + StaticSSHPort bool `json:"staticSSHPort"` + SkipSocketForwarding bool `json:"skipSocketForwarding"` + NoCloudInit bool `json:"noCloudInit"` + RosettaEnabled bool `json:"rosettaEnabled"` + RosettaBinFmt bool `json:"rosettaBinFmt"` + SupportedImageFormats []string `json:"supportedImageFormats,omitempty"` } diff --git a/pkg/driver/krunkit/krunkit_driver_darwin_arm64.go b/pkg/driver/krunkit/krunkit_driver_darwin_arm64.go index 0e2cb66e..e76caa83 100644 --- a/pkg/driver/krunkit/krunkit_driver_darwin_arm64.go +++ b/pkg/driver/krunkit/krunkit_driver_darwin_arm64.go @@ -335,9 +335,10 @@ func (l *LimaKrunkitDriver) Info(_ context.Context) driver.Info { } info.Features = driver.DriverFeatures{ - DynamicSSHAddress: false, - SkipSocketForwarding: false, - CanRunGUI: false, + DynamicSSHAddress: false, + SkipSocketForwarding: false, + CanRunGUI: false, + SupportedImageFormats: []string{string(raw.Type)}, } return info } diff --git a/pkg/driver/qemu/qemu.go b/pkg/driver/qemu/qemu.go index 01bee6b8..1db385b5 100644 --- a/pkg/driver/qemu/qemu.go +++ b/pkg/driver/qemu/qemu.go @@ -610,7 +610,7 @@ func Cmdline(ctx context.Context, cfg Config) (exe string, args []string, err er switch f.VMType { case "", limatype.QEMU: if f.Arch == *y.Arch { - if _, err = fileutils.DownloadFile(ctx, downloadedFirmware, f.File, true, "UEFI code "+f.Location, *y.Arch); err != nil { + if _, err = fileutils.DownloadFile(ctx, downloadedFirmware, f.File, true, "UEFI code "+f.Location, *y.Arch, nil); err != nil { logrus.WithError(err).Warnf("failed to download %#q", f.Location) continue loop } diff --git a/pkg/driver/qemu/qemu_driver.go b/pkg/driver/qemu/qemu_driver.go index 98eb0a4e..5d23025b 100644 --- a/pkg/driver/qemu/qemu_driver.go +++ b/pkg/driver/qemu/qemu_driver.go @@ -26,6 +26,10 @@ import ( "github.com/coreos/go-semver/semver" "github.com/digitalocean/go-qemu/qmp" "github.com/digitalocean/go-qemu/qmp/raw" + "github.com/lima-vm/go-qcow2reader/image/qcow2" + rawImage "github.com/lima-vm/go-qcow2reader/image/raw" + "github.com/lima-vm/go-qcow2reader/image/vhdx" + "github.com/lima-vm/go-qcow2reader/image/vmdk" "github.com/sirupsen/logrus" "github.com/lima-vm/lima/v2/pkg/driver" @@ -866,9 +870,10 @@ func (l *LimaQemuDriver) Info(_ context.Context) driver.Info { info.VsockPort = l.vSockPort info.Features = driver.DriverFeatures{ - DynamicSSHAddress: false, - SkipSocketForwarding: false, - CanRunGUI: false, + DynamicSSHAddress: false, + SkipSocketForwarding: false, + CanRunGUI: false, + SupportedImageFormats: []string{string(qcow2.Type), string(rawImage.Type), string(vmdk.Type), string(vhdx.Type)}, // Not a comprehensive list of supported formats } return info } diff --git a/pkg/driver/vz/vz_driver_darwin.go b/pkg/driver/vz/vz_driver_darwin.go index 873cc389..82a45746 100644 --- a/pkg/driver/vz/vz_driver_darwin.go +++ b/pkg/driver/vz/vz_driver_darwin.go @@ -559,11 +559,12 @@ func (l *LimaVzDriver) Info(_ context.Context) driver.Info { guiFlag = l.canRunGUI() } info.Features = driver.DriverFeatures{ - DynamicSSHAddress: false, - SkipSocketForwarding: false, - CanRunGUI: guiFlag, - RosettaEnabled: l.rosettaEnabled, - RosettaBinFmt: l.rosettaBinFmt, + DynamicSSHAddress: false, + SkipSocketForwarding: false, + CanRunGUI: guiFlag, + RosettaEnabled: l.rosettaEnabled, + RosettaBinFmt: l.rosettaBinFmt, + SupportedImageFormats: []string{string(raw.Type), string(asif.Type)}, } return info } diff --git a/pkg/driver/wsl2/fs.go b/pkg/driver/wsl2/fs.go index 953bde5f..4ded42d8 100644 --- a/pkg/driver/wsl2/fs.go +++ b/pkg/driver/wsl2/fs.go @@ -23,7 +23,7 @@ func EnsureFs(ctx context.Context, inst *limatype.Instance) error { var ensuredBaseDisk bool errs := make([]error, len(inst.Config.Images)) for i, f := range inst.Config.Images { - if _, err := fileutils.DownloadFile(ctx, baseDisk, f.File, true, "the image", *inst.Config.Arch); err != nil { + if _, err := fileutils.DownloadFile(ctx, baseDisk, f.File, true, "the image", *inst.Config.Arch, nil); err != nil { errs[i] = err continue } diff --git a/pkg/driverutil/disk.go b/pkg/driverutil/disk.go index 432c518d..58151146 100644 --- a/pkg/driverutil/disk.go +++ b/pkg/driverutil/disk.go @@ -14,6 +14,7 @@ import ( "github.com/lima-vm/go-qcow2reader/image" "github.com/sirupsen/logrus" + "github.com/lima-vm/lima/v2/pkg/imgutil/nativeimgutil" "github.com/lima-vm/lima/v2/pkg/imgutil/proxyimgutil" "github.com/lima-vm/lima/v2/pkg/iso9660util" "github.com/lima-vm/lima/v2/pkg/limatype/filenames" @@ -95,10 +96,25 @@ func EnsureDisk(ctx context.Context, instDir, diskSize string, diskImageFormat i return fmt.Errorf("failed to create disk %#q: %w", diskPath, err) } } else { - if err := diskUtil.Convert(ctx, diskImageFormat, imagePath, diskPath, &diskSizeInBytes, false); err != nil { - return fmt.Errorf("failed to convert %#q to %#q: %w", imagePath, diskPath, err) + format, err := nativeimgutil.DetectFormat(imagePath) + if err != nil { + return err + } + + // Resize handled by prepareDisk() after CreateDisk() + if format == string(diskImageFormat) { + if err = os.Rename(imagePath, diskPath); err != nil { + return fmt.Errorf("failed to rename %#q to %#q: %w", imagePath, diskPath, err) + } + if err := os.Chmod(diskPath, 0o644); err != nil { + return fmt.Errorf("failed to chmod %#q: %w", diskPath, err) + } + } else { + if err := diskUtil.Convert(ctx, diskImageFormat, imagePath, diskPath, &diskSizeInBytes, false); err != nil { + return fmt.Errorf("failed to convert %#q to %#q: %w", imagePath, diskPath, err) + } + os.Remove(imagePath) } - os.Remove(imagePath) } return nil } diff --git a/pkg/fileutils/download.go b/pkg/fileutils/download.go index 2166baae..d9a0f435 100644 --- a/pkg/fileutils/download.go +++ b/pkg/fileutils/download.go @@ -19,18 +19,23 @@ import ( var ErrSkipped = errors.New("skipped to download") // DownloadFile downloads a file to the cache, optionally copying it to the destination. Returns path in cache. -func DownloadFile(ctx context.Context, dest string, f limatype.File, decompress bool, description string, expectedArch limatype.Arch) (string, error) { +func DownloadFile(ctx context.Context, dest string, f limatype.File, decompress bool, description string, expectedArch limatype.Arch, supportedImageFormats []string) (string, error) { if f.Arch != expectedArch { return "", fmt.Errorf("%w: %#q: unsupported arch: %#q", ErrSkipped, f.Location, f.Arch) } fields := logrus.Fields{"location": f.Location, "arch": f.Arch, "digest": f.Digest} logrus.WithFields(fields).Infof("Attempting to download %s", description) - res, err := downloader.Download(ctx, dest, f.Location, + opts := []downloader.Opt{ downloader.WithCache(), downloader.WithDecompress(decompress), downloader.WithDescription(fmt.Sprintf("%s (%s)", description, path.Base(f.Location))), downloader.WithExpectedDigest(f.Digest), - ) + } + if len(supportedImageFormats) > 0 { + opts = append(opts, downloader.WithImageFormats(supportedImageFormats)) + } + + res, err := downloader.Download(ctx, dest, f.Location, opts...) if err != nil { return "", fmt.Errorf("failed to download %#q: %w", f.Location, err) } diff --git a/pkg/imgutil/nativeimgutil/nativeimgutil.go b/pkg/imgutil/nativeimgutil/nativeimgutil.go index b421ff72..61616d32 100644 --- a/pkg/imgutil/nativeimgutil/nativeimgutil.go +++ b/pkg/imgutil/nativeimgutil/nativeimgutil.go @@ -216,6 +216,22 @@ func makeSparse(f *os.File, offset int64) error { return f.Truncate(offset) } +func DetectFormat(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + + img, err := qcow2reader.Open(f) + if err != nil { + return "", err + } + defer img.Close() + + return string(img.Type()), nil +} + // CreateDisk creates a new disk image with the specified size. func (n *NativeImageUtil) CreateDisk(_ context.Context, disk string, size int64) error { if _, err := os.Stat(disk); err == nil || !errors.Is(err, fs.ErrNotExist) { diff --git a/pkg/instance/start.go b/pkg/instance/start.go index 1f7c7ecd..8b3f15b3 100644 --- a/pkg/instance/start.go +++ b/pkg/instance/start.go @@ -84,6 +84,8 @@ func Prepare(ctx context.Context, inst *limatype.Instance, guestAgent string) (* return nil, err } + supportedImageFormats := limaDriver.Info(ctx).Features.SupportedImageFormats + created := limayaml.IsExistingInstanceDir(inst.Dir) imagePath := filepath.Join(inst.Dir, filenames.Image) @@ -112,13 +114,13 @@ func Prepare(ctx context.Context, inst *limatype.Instance, guestAgent string) (* errs[i] = fmt.Errorf("%w: variant mismatch (expected %q, got %q)", fileutils.ErrSkipped, targetVariant, f.Variant) continue } - if _, err := fileutils.DownloadFile(ctx, imagePath, f.File, true, "the image", *inst.Config.Arch); err != nil { + if _, err := fileutils.DownloadFile(ctx, imagePath, f.File, true, "the image", *inst.Config.Arch, supportedImageFormats); err != nil { errs[i] = err continue } if f.Kernel != nil { // ensure decompress kernel because vz expects it to be decompressed - if _, err := fileutils.DownloadFile(ctx, kernel, f.Kernel.File, true, "the kernel", *inst.Config.Arch); err != nil { + if _, err := fileutils.DownloadFile(ctx, kernel, f.Kernel.File, true, "the kernel", *inst.Config.Arch, nil); err != nil { errs[i] = err continue } @@ -131,7 +133,7 @@ func Prepare(ctx context.Context, inst *limatype.Instance, guestAgent string) (* } if f.Initrd != nil { // vz does not need initrd to be decompressed - if _, err := fileutils.DownloadFile(ctx, initrd, *f.Initrd, false, "the initrd", *inst.Config.Arch); err != nil { + if _, err := fileutils.DownloadFile(ctx, initrd, *f.Initrd, false, "the initrd", *inst.Config.Arch, nil); err != nil { errs[i] = err continue } @@ -155,7 +157,7 @@ func Prepare(ctx context.Context, inst *limatype.Instance, guestAgent string) (* errs := make([]error, len(winOpts.VirtioWin)) virtioWin := filepath.Join(inst.Dir, filenames.VirtioWin) for i, f := range winOpts.VirtioWin { - if _, err := fileutils.DownloadFile(ctx, virtioWin, f, true, "virtio-win", *inst.Config.Arch); err != nil { + if _, err := fileutils.DownloadFile(ctx, virtioWin, f, true, "virtio-win", *inst.Config.Arch, nil); err != nil { errs[i] = err continue } diff --git a/website/content/en/docs/dev/internals.md b/website/content/en/docs/dev/internals.md index 89dc8e76..40dfeb21 100644 --- a/website/content/en/docs/dev/internals.md +++ b/website/content/en/docs/dev/internals.md @@ -145,6 +145,9 @@ The directory contains the following files: - `data`: data - `.digest`: digest of the data, in OCI format. e.g., file name `sha256.digest`, with content `sha256:5ba3d476707d510fe3ca3928e9cda5d0b4ce527d42b343404c92d563f82ba967` +- `imgconv/`: (Optional) a converted copy of the image, in a format the current driver can use. The downloader creates this on demand when the driver reports its `SupportedImageFormats` and the cached `data` is not one of them (for example, a `qcow2` image requested by the `vz` driver, which only boots raw). The original `data` and its digest file are left as-is, so another driver can still use them. + - `raw`: the converted raw image. If `data` is newer than `raw`, the raw copy is treated as stale and rebuilt. + - `raw.digest`: digest of the raw image. Uses the same algorithm as the digest the caller passed in, and is only read back when the caller passes an expected digest. ## Ansible