diff --git a/docs/resources/virtual_environment_container.md b/docs/resources/virtual_environment_container.md index aea8a010..accd2ea7 100644 --- a/docs/resources/virtual_environment_container.md +++ b/docs/resources/virtual_environment_container.md @@ -42,7 +42,7 @@ resource "proxmox_virtual_environment_container" "ubuntu_container" { } operating_system { - template_file_id = proxmox_virtual_environment_file.ubuntu_container_template.id + template_file_id = proxmox_virtual_environment_file.latest_ubuntu_22_jammy_lxc_img.id type = "ubuntu" } @@ -61,14 +61,11 @@ resource "proxmox_virtual_environment_container" "ubuntu_container" { } -resource "proxmox_virtual_environment_file" "ubuntu_container_template" { +resource "proxmox_virtual_environment_download_file" "latest_ubuntu_22_jammy_lxc_img" { content_type = "vztmpl" datastore_id = "local" node_name = "first-node" - - source_file { - path = "http://download.proxmox.com/images/system/ubuntu-20.04-standard_20.04-1_amd64.tar.gz" - } + url = "https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.tar.gz" } resource "random_password" "ubuntu_container_password" { diff --git a/docs/resources/virtual_environment_download_file.md b/docs/resources/virtual_environment_download_file.md new file mode 100644 index 00000000..464816e7 --- /dev/null +++ b/docs/resources/virtual_environment_download_file.md @@ -0,0 +1,91 @@ +--- +layout: page +title: proxmox_virtual_environment_download_file +parent: Resources +subcategory: Virtual Environment +description: |- + Manages files upload using PVE download-url API. It can be fully compatible and faster replacement for image files created using proxmox_virtual_environment_file. Supports images for VMs (ISO images) and LXC (CT Templates). +--- + +# Resource: proxmox_virtual_environment_download_file + +Manages files upload using PVE download-url API. It can be fully compatible and faster replacement for image files created using `proxmox_virtual_environment_file`. Supports images for VMs (ISO images) and LXC (CT Templates). + +## Example Usage + +```terraform +resource "proxmox_virtual_environment_download_file" "release_20231228_debian_12_bookworm_qcow2_img" { + content_type = "iso" + datastore_id = "local" + file_name = "debian-12-generic-amd64-20231228-1609.img" + node_name = "pve" + url = "https://cloud.debian.org/images/cloud/bookworm/20231228-1609/debian-12-generic-amd64-20231228-1609.qcow2" + checksum = "d2fbcf11fb28795842e91364d8c7b69f1870db09ff299eb94e4fbbfa510eb78d141e74c1f4bf6dfa0b7e33d0c3b66e6751886feadb4e9916f778bab1776bdf1b" + checksum_algorithm = "sha512" +} + +resource "proxmox_virtual_environment_download_file" "latest_debian_12_bookworm_qcow2_img" { + content_type = "iso" + datastore_id = "local" + file_name = "debian-12-generic-amd64.qcow2.img" + node_name = "pve" + url = "https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2" +} + +resource "proxmox_virtual_environment_download_file" "latest_ubuntu_22_jammy_qcow2_img" { + content_type = "iso" + datastore_id = "local" + node_name = "pve" + url = "https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img" +} + +resource "proxmox_virtual_environment_download_file" "latest_static_ubuntu_24_noble_qcow2_img" { + content_type = "iso" + datastore_id = "local" + node_name = "pve" + url = "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img" + overwrite = false +} + +resource "proxmox_virtual_environment_download_file" "release_20231211_ubuntu_22_jammy_lxc_img" { + content_type = "vztmpl" + datastore_id = "local" + node_name = "pve" + url = "https://cloud-images.ubuntu.com/releases/22.04/release-20231211/ubuntu-22.04-server-cloudimg-amd64-root.tar.xz" + checksum = "c9997dcfea5d826fd04871f960c513665f2e87dd7450bba99f68a97e60e4586e" + checksum_algorithm = "sha256" + upload_timeout = 4444 +} + +resource "proxmox_virtual_environment_download_file" "latest_ubuntu_22_jammy_lxc_img" { + content_type = "vztmpl" + datastore_id = "local" + node_name = "pve" + url = "https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.tar.gz" +} +``` + + +## Schema + +### Required + +- `content_type` (String) The file content type. Must be `iso` for VM images or `vztmpl` for LXC images. +- `datastore_id` (String) The identifier for the target datastore. +- `node_name` (String) The node name. +- `url` (String) The URL to download the file from. Format `https?://.*`. + +### Optional + +- `checksum` (String) The expected checksum of the file. +- `checksum_algorithm` (String) The algorithm to calculate the checksum of the file. Must be `md5` | `sha1` | `sha224` | `sha256` | `sha384` | `sha512`. +- `decompression_algorithm` (String) Decompress the downloaded file using the specified compression algorithm. Must be one of `gz` | `lzo` | `zst`. +- `file_name` (String) The file name. If not provided, it is calculated using `url`. PVE will raise 'wrong file extension' error for some popular extensions file `.raw` or `.qcow2`. Workaround is to use e.g. `.img` instead. +- `overwrite` (Boolean) If `true` and size of uploaded file is different, than size from `url` Content-Length header, file will be downloaded again. If `false`, there will be no checks. +- `upload_timeout` (Number) The file download timeout seconds. Default is 600 (10min). +- `verify` (Boolean) By default `true`. If `false`, no SSL/TLS certificates will be verified. + +### Read-Only + +- `id` (String) The unique identifier of this resource. +- `size` (Number) The file size. diff --git a/docs/resources/virtual_environment_file.md b/docs/resources/virtual_environment_file.md index 96b670ec..6de7f692 100644 --- a/docs/resources/virtual_environment_file.md +++ b/docs/resources/virtual_environment_file.md @@ -31,6 +31,8 @@ resource "proxmox_virtual_environment_file" "backup" { ### Images +**Consider using `proxmox_environment_download_file` resource instead. Using this resource for images is less efficient (requires to transfer uploaded image to node) though still supported.** + ```terraform resource "proxmox_virtual_environment_file" "ubuntu_container_template" { content_type = "iso" @@ -82,6 +84,8 @@ EOF ### Container Template (`vztmpl`) +**Consider using `proxmox_environment_download_file` resource instead. Using this resource for container images is less efficient (requires to transfer uploaded image to node) though still supported.** + ```terraform resource "proxmox_virtual_environment_file" "ubuntu_container_template" { content_type = "vztmpl" diff --git a/docs/resources/virtual_environment_vm.md b/docs/resources/virtual_environment_vm.md index 3fb0967e..b9a02b1e 100644 --- a/docs/resources/virtual_environment_vm.md +++ b/docs/resources/virtual_environment_vm.md @@ -37,7 +37,7 @@ resource "proxmox_virtual_environment_vm" "ubuntu_vm" { disk { datastore_id = "local-lvm" - file_id = proxmox_virtual_environment_file.ubuntu_cloud_image.id + file_id = proxmox_virtual_environment_file.latest_ubuntu_22_jammy_qcow2_img.id interface = "scsi0" } @@ -72,14 +72,11 @@ resource "proxmox_virtual_environment_vm" "ubuntu_vm" { serial_device {} } -resource "proxmox_virtual_environment_file" "ubuntu_cloud_image" { +resource "proxmox_virtual_environment_download_file" "latest_ubuntu_22_jammy_qcow2_img" { content_type = "iso" datastore_id = "local" - node_name = "first-node" - - source_file { - path = "http://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img" - } + node_name = "pve" + url = "https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img" } resource "random_password" "ubuntu_vm_password" { diff --git a/example/resource_virtual_environment_container.tf b/example/resource_virtual_environment_container.tf index cc181813..98409085 100644 --- a/example/resource_virtual_environment_container.tf +++ b/example/resource_virtual_environment_container.tf @@ -42,7 +42,7 @@ resource "proxmox_virtual_environment_container" "example_template" { node_name = data.proxmox_virtual_environment_nodes.example.names[0] operating_system { - template_file_id = proxmox_virtual_environment_file.ubuntu_container_template.id + template_file_id = proxmox_virtual_environment_download_file.release_20231211_ubuntu_22_jammy_lxc_img.id type = "ubuntu" } diff --git a/example/resource_virtual_environment_download_file.tf b/example/resource_virtual_environment_download_file.tf new file mode 100644 index 00000000..ba592402 --- /dev/null +++ b/example/resource_virtual_environment_download_file.tf @@ -0,0 +1,20 @@ +## Debian and ubuntu image download + +resource "proxmox_virtual_environment_download_file" "release_20231211_ubuntu_22_jammy_lxc_img" { + content_type = "vztmpl" + datastore_id = "local" + node_name = "pve" + url = "https://cloud-images.ubuntu.com/releases/22.04/release-20231211/ubuntu-22.04-server-cloudimg-amd64-root.tar.xz" + checksum = "c9997dcfea5d826fd04871f960c513665f2e87dd7450bba99f68a97e60e4586e" + checksum_algorithm = "sha256" + upload_timeout = 4444 +} + +resource "proxmox_virtual_environment_download_file" "latest_debian_12_bookworm_qcow2_img" { + content_type = "iso" + datastore_id = "local" + file_name = "debian-12-generic-amd64.img" + node_name = "pve" + url = "https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2" + overwrite = true +} diff --git a/example/resource_virtual_environment_vm.tf b/example/resource_virtual_environment_vm.tf index ec6d17a5..d7a17b3d 100644 --- a/example/resource_virtual_environment_vm.tf +++ b/example/resource_virtual_environment_vm.tf @@ -48,7 +48,7 @@ resource "proxmox_virtual_environment_vm" "example_template" { disk { datastore_id = local.datastore_id - file_id = proxmox_virtual_environment_file.ubuntu_cloud_image.id + file_id = proxmox_virtual_environment_download_file.latest_debian_12_bookworm_qcow2_img.id interface = "scsi0" discard = "on" cache = "writeback" diff --git a/examples/resources/proxmox_virtual_environment_download_file/resource.tf b/examples/resources/proxmox_virtual_environment_download_file/resource.tf new file mode 100644 index 00000000..655f64e2 --- /dev/null +++ b/examples/resources/proxmox_virtual_environment_download_file/resource.tf @@ -0,0 +1,49 @@ +resource "proxmox_virtual_environment_download_file" "release_20231228_debian_12_bookworm_qcow2_img" { + content_type = "iso" + datastore_id = "local" + file_name = "debian-12-generic-amd64-20231228-1609.img" + node_name = "pve" + url = "https://cloud.debian.org/images/cloud/bookworm/20231228-1609/debian-12-generic-amd64-20231228-1609.qcow2" + checksum = "d2fbcf11fb28795842e91364d8c7b69f1870db09ff299eb94e4fbbfa510eb78d141e74c1f4bf6dfa0b7e33d0c3b66e6751886feadb4e9916f778bab1776bdf1b" + checksum_algorithm = "sha512" +} + +resource "proxmox_virtual_environment_download_file" "latest_debian_12_bookworm_qcow2_img" { + content_type = "iso" + datastore_id = "local" + file_name = "debian-12-generic-amd64.qcow2.img" + node_name = "pve" + url = "https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2" +} + +resource "proxmox_virtual_environment_download_file" "latest_ubuntu_22_jammy_qcow2_img" { + content_type = "iso" + datastore_id = "local" + node_name = "pve" + url = "https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img" +} + +resource "proxmox_virtual_environment_download_file" "latest_static_ubuntu_24_noble_qcow2_img" { + content_type = "iso" + datastore_id = "local" + node_name = "pve" + url = "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img" + overwrite = false +} + +resource "proxmox_virtual_environment_download_file" "release_20231211_ubuntu_22_jammy_lxc_img" { + content_type = "vztmpl" + datastore_id = "local" + node_name = "pve" + url = "https://cloud-images.ubuntu.com/releases/22.04/release-20231211/ubuntu-22.04-server-cloudimg-amd64-root.tar.xz" + checksum = "c9997dcfea5d826fd04871f960c513665f2e87dd7450bba99f68a97e60e4586e" + checksum_algorithm = "sha256" + upload_timeout = 4444 +} + +resource "proxmox_virtual_environment_download_file" "latest_ubuntu_22_jammy_lxc_img" { + content_type = "vztmpl" + datastore_id = "local" + node_name = "pve" + url = "https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.tar.gz" +} diff --git a/fwprovider/provider.go b/fwprovider/provider.go index 7b1dbe04..82fb12b3 100644 --- a/fwprovider/provider.go +++ b/fwprovider/provider.go @@ -381,6 +381,7 @@ func (p *proxmoxProvider) Resources(_ context.Context) []func() resource.Resourc NewClusterOptionsResource, NewLinuxBridgeResource, NewLinuxVLANResource, + NewDownloadFileResource, } } diff --git a/fwprovider/resource_download_file.go b/fwprovider/resource_download_file.go new file mode 100644 index 00000000..af700a27 --- /dev/null +++ b/fwprovider/resource_download_file.go @@ -0,0 +1,609 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +package fwprovider + +import ( + "context" + "fmt" + "regexp" + "strconv" + "strings" + + "github.com/bpg/terraform-provider-proxmox/fwprovider/structure" + "github.com/hashicorp/terraform-plugin-framework-validators/stringvalidator" + "github.com/hashicorp/terraform-plugin-framework/path" + "github.com/hashicorp/terraform-plugin-framework/resource" + "github.com/hashicorp/terraform-plugin-framework/resource/schema" + "github.com/hashicorp/terraform-plugin-framework/resource/schema/booldefault" + "github.com/hashicorp/terraform-plugin-framework/resource/schema/int64default" + "github.com/hashicorp/terraform-plugin-framework/resource/schema/int64planmodifier" + "github.com/hashicorp/terraform-plugin-framework/resource/schema/planmodifier" + "github.com/hashicorp/terraform-plugin-framework/resource/schema/stringplanmodifier" + "github.com/hashicorp/terraform-plugin-framework/schema/validator" + "github.com/hashicorp/terraform-plugin-framework/types" + + "github.com/bpg/terraform-provider-proxmox/proxmox" + "github.com/bpg/terraform-provider-proxmox/proxmox/nodes" + "github.com/bpg/terraform-provider-proxmox/proxmox/nodes/storage" + proxmoxtypes "github.com/bpg/terraform-provider-proxmox/proxmox/types" +) + +var ( + _ resource.Resource = &downloadFileResource{} + _ resource.ResourceWithConfigure = &downloadFileResource{} + httpRe = regexp.MustCompile(`https?://.*`) +) + +func sizeRequiresReplace() planmodifier.Int64 { + return sizeRequiresReplaceModifier{} +} + +type sizeRequiresReplaceModifier struct{} + +func (r sizeRequiresReplaceModifier) PlanModifyInt64( + ctx context.Context, + req planmodifier.Int64Request, + resp *planmodifier.Int64Response, +) { + // Do not replace on resource creation. + if req.State.Raw.IsNull() { + return + } + + // // Do not replace on resource destroy. + if req.Plan.Raw.IsNull() { + return + } + + var plan, state downloadFileModel + + resp.Diagnostics.Append(req.State.Get(ctx, &state)...) + resp.Diagnostics.Append(req.Plan.Get(ctx, &plan)...) + + originalStateSizeBytes, diags := req.Private.GetKey(ctx, "original_state_size") + + resp.Diagnostics.Append(diags...) + + if originalStateSizeBytes != nil { + originalStateSize, err := strconv.ParseInt(string(originalStateSizeBytes), 10, 64) + if err != nil { + resp.Diagnostics.AddError( + "Unexpected error when reading originalStateSize from Private", + fmt.Sprintf( + "Unexpected error in ParseInt: %s", + err.Error(), + ), + ) + + return + } + + if state.Size.ValueInt64() != originalStateSize { + resp.RequiresReplace = true + resp.PlanValue = types.Int64Value(originalStateSize) + + resp.Diagnostics.AddWarning( + "The file size in datastore has changed.", + fmt.Sprintf( + "Previous size %d does not match size from datastore: %d", + originalStateSize, + state.Size.ValueInt64(), + ), + ) + + return + } + } + + urlSizeBytes, diags := req.Private.GetKey(ctx, "url_size") + + resp.Diagnostics.Append(diags...) + + if (urlSizeBytes != nil) && (plan.URL.ValueString() == state.URL.ValueString()) { + urlSize, err := strconv.ParseInt(string(urlSizeBytes), 10, 64) + if err != nil { + resp.Diagnostics.AddError( + "Unexpected error when reading urlSize from Private", + fmt.Sprintf( + "Unexpected error in ParseInt: %s", + err.Error(), + ), + ) + + return + } + + if state.Size.ValueInt64() != urlSize { + resp.RequiresReplace = true + resp.PlanValue = types.Int64Value(urlSize) + + resp.Diagnostics.AddWarning( + "The file size from url has changed.", + fmt.Sprintf( + "Size from url %d does not match size from datastore: %d", + urlSize, + state.Size.ValueInt64(), + ), + ) + + return + } + } +} + +func (r sizeRequiresReplaceModifier) Description(_ context.Context) string { + return "Triggers resource force replacement if `size` in state does not match remote value." +} + +func (r sizeRequiresReplaceModifier) MarkdownDescription(_ context.Context) string { + return "Triggers resource force replacement if `size` in state does not match remote value." +} + +type downloadFileModel struct { + ID types.String `tfsdk:"id"` + Content types.String `tfsdk:"content_type"` + FileName types.String `tfsdk:"file_name"` + Storage types.String `tfsdk:"datastore_id"` + Node types.String `tfsdk:"node_name"` + Size types.Int64 `tfsdk:"size"` + URL types.String `tfsdk:"url"` + Checksum types.String `tfsdk:"checksum"` + DecompressionAlgorithm types.String `tfsdk:"decompression_algorithm"` + UploadTimeout types.Int64 `tfsdk:"upload_timeout"` + ChecksumAlgorithm types.String `tfsdk:"checksum_algorithm"` + Verify types.Bool `tfsdk:"verify"` + Overwrite types.Bool `tfsdk:"overwrite"` +} + +// NewDownloadFileResource manages files downloaded using proxmomx API. +func NewDownloadFileResource() resource.Resource { + return &downloadFileResource{} +} + +type downloadFileResource struct { + client proxmox.Client +} + +func (r *downloadFileResource) Metadata( + _ context.Context, + req resource.MetadataRequest, + resp *resource.MetadataResponse, +) { + resp.TypeName = req.ProviderTypeName + "_download_file" +} + +// Schema defines the schema for the resource. +func (r *downloadFileResource) Schema( + _ context.Context, + _ resource.SchemaRequest, + resp *resource.SchemaResponse, +) { + resp.Schema = schema.Schema{ + Description: "Manages files upload using PVE download-url API. ", + MarkdownDescription: "Manages files upload using PVE download-url API. " + + "It can be fully compatible and faster replacement for image files created using " + + "`proxmox_virtual_environment_file`. Supports images for VMs (ISO images) and LXC (CT Templates).", + Attributes: map[string]schema.Attribute{ + "id": structure.IDAttribute(), + "content_type": schema.StringAttribute{ + Description: "The file content type. Must be `iso` for VM images or `vztmpl` for LXC images.", + Required: true, + Validators: []validator.String{stringvalidator.OneOf([]string{ + "iso", + "vztmpl", + }...)}, + PlanModifiers: []planmodifier.String{ + stringplanmodifier.RequiresReplace(), + }, + }, + "file_name": schema.StringAttribute{ + Description: "The file name. If not provided, it is calculated " + + "using `url`. PVE will raise 'wrong file extension' error for some popular " + + "extensions file `.raw` or `.qcow2`. Workaround is to use e.g. `.img` instead.", + Computed: true, + Required: false, + Optional: true, + PlanModifiers: []planmodifier.String{ + stringplanmodifier.RequiresReplace(), + stringplanmodifier.UseStateForUnknown(), + }, + }, + "datastore_id": schema.StringAttribute{ + Description: "The identifier for the target datastore.", + Required: true, + PlanModifiers: []planmodifier.String{ + stringplanmodifier.RequiresReplace(), + }, + }, + "node_name": schema.StringAttribute{ + Description: "The node name.", + Required: true, + PlanModifiers: []planmodifier.String{ + stringplanmodifier.RequiresReplace(), + }, + }, + "size": schema.Int64Attribute{ + Description: "The file size.", + Optional: false, + Required: false, + Computed: true, + PlanModifiers: []planmodifier.Int64{ + int64planmodifier.UseStateForUnknown(), + int64planmodifier.RequiresReplace(), + sizeRequiresReplace(), + }, + }, + "upload_timeout": schema.Int64Attribute{ + Description: "The file download timeout seconds. Default is 600 (10min).", + Optional: true, + Computed: true, + Default: int64default.StaticInt64(600), + }, + "url": schema.StringAttribute{ + Description: "The URL to download the file from. Format `https?://.*`.", + Required: true, + Validators: []validator.String{ + stringvalidator.RegexMatches(httpRe, "Must match http url regex"), + }, + PlanModifiers: []planmodifier.String{ + stringplanmodifier.RequiresReplace(), + }, + }, + "checksum": schema.StringAttribute{ + Description: "The expected checksum of the file.", + Optional: true, + Default: nil, + PlanModifiers: []planmodifier.String{ + stringplanmodifier.RequiresReplace(), + }, + Validators: []validator.String{ + stringvalidator.AlsoRequires(path.MatchRoot("checksum_algorithm")), + }, + }, + "decompression_algorithm": schema.StringAttribute{ + Description: "Decompress the downloaded file using the " + + "specified compression algorithm. Must be one of `gz` | `lzo` | `zst`.", + Optional: true, + Default: nil, + Validators: []validator.String{ + stringvalidator.OneOf([]string{ + "gz", + "lzo", + "zst", + }...), + }, + }, + "checksum_algorithm": schema.StringAttribute{ + Description: "The algorithm to calculate the checksum of the file. " + + "Must be `md5` | `sha1` | `sha224` | `sha256` | `sha384` | `sha512`.", + Optional: true, + Validators: []validator.String{ + stringvalidator.OneOf([]string{ + "md5", + "sha1", + "sha224", + "sha256", + "sha384", + "sha512", + }...), + stringvalidator.AlsoRequires(path.MatchRoot("checksum")), + }, + Default: nil, + PlanModifiers: []planmodifier.String{ + stringplanmodifier.RequiresReplace(), + }, + }, + "verify": schema.BoolAttribute{ + Description: "By default `true`. If `false`, no SSL/TLS certificates will be verified.", + Optional: true, + Computed: true, + Default: booldefault.StaticBool(true), + }, + "overwrite": schema.BoolAttribute{ + Description: "If `true` and size of uploaded file is different, " + + "than size from `url` Content-Length header, file will be downloaded again. " + + "If `false`, there will be no checks.", + Optional: true, + Computed: true, + Default: booldefault.StaticBool(true), + }, + }, + } +} + +func (r *downloadFileResource) Configure( + _ context.Context, + req resource.ConfigureRequest, + resp *resource.ConfigureResponse, +) { + if req.ProviderData == nil { + return + } + + client, ok := req.ProviderData.(proxmox.Client) + + if !ok { + resp.Diagnostics.AddError( + "Unexpected Resource Configure Type", + fmt.Sprintf("Expected *proxmox.Client, got: %T", req.ProviderData), + ) + + return + } + + r.client = client +} + +func (r *downloadFileResource) Create( + ctx context.Context, + req resource.CreateRequest, + resp *resource.CreateResponse, +) { + var plan downloadFileModel + diags := req.Plan.Get(ctx, &plan) + resp.Diagnostics.Append(diags...) + + if resp.Diagnostics.HasError() { + return + } + + fileMetadata, err := r.getURLMetadata( + ctx, + &plan, + ) + if err != nil { + resp.Diagnostics.AddError( + "Error initiating file download", + "Could not get file metadata, unexpected error: "+err.Error(), + ) + + return + } + + if plan.FileName.IsUnknown() { + plan.FileName = types.StringValue(*fileMetadata.Filename) + } + + nodesClient := r.client.Node(plan.Node.ValueString()) + verify := proxmoxtypes.CustomBool(plan.Verify.ValueBool()) + + downloadFileReq := storage.DownloadURLPostRequestBody{ + Node: plan.Node.ValueStringPointer(), + Storage: plan.Storage.ValueStringPointer(), + Content: plan.Content.ValueStringPointer(), + Checksum: plan.Checksum.ValueStringPointer(), + ChecksumAlgorithm: plan.ChecksumAlgorithm.ValueStringPointer(), + Compression: plan.DecompressionAlgorithm.ValueStringPointer(), + FileName: plan.FileName.ValueStringPointer(), + URL: plan.URL.ValueStringPointer(), + Verify: &verify, + } + + storageClient := nodesClient.Storage(plan.Storage.ValueString()) + err = storageClient.DownloadFileByURL( + ctx, + &downloadFileReq, + plan.UploadTimeout.ValueInt64(), + ) + + if err != nil { + if strings.Contains(err.Error(), "refusing to override existing file") { + resp.Diagnostics.AddError( + "File already exists in a datastore, it was created outside of Terraform "+ + "or is managed by another resource.", + fmt.Sprintf( + "File already exists in a datastore: `%s`, "+ + "error: %s", + plan.FileName.ValueString(), + err.Error(), + ), + ) + } else { + resp.Diagnostics.AddError( + "Error creating Download File interface", + fmt.Sprintf( + "Could not DownloadFileByURL: `%s`, "+ + "unexpected error: %s", + plan.FileName.ValueString(), + err.Error(), + ), + ) + } + + return + } + + plan.ID = types.StringValue(plan.Storage.ValueString() + ":" + + plan.Content.ValueString() + "/" + plan.FileName.ValueString()) + + err = r.read(ctx, &plan) + + if err != nil { + resp.Diagnostics.AddError( + "Error when reading file from datastore", err.Error(), + ) + } + + resp.State.Set(ctx, plan) + resp.Diagnostics.Append(diags...) +} + +func (r *downloadFileResource) getURLMetadata( + ctx context.Context, + model *downloadFileModel, +) (*nodes.QueryURLMetadataGetResponseData, error) { + nodesClient := r.client.Node(model.Node.ValueString()) + verify := proxmoxtypes.CustomBool(model.Verify.ValueBool()) + + queryURLMetadataReq := nodes.QueryURLMetadataGetRequestBody{ + URL: model.URL.ValueStringPointer(), + Verify: &verify, + } + + fileMetadata, err := nodesClient.GetQueryURLMetadata( + ctx, + &queryURLMetadataReq, + ) + if err != nil { + return nil, fmt.Errorf( + "error fetching metadata from download url, "+ + "unexpected error in GetQueryURLMetadata: %w", + err, + ) + } + + return fileMetadata, nil +} + +func (r *downloadFileResource) read( + ctx context.Context, + model *downloadFileModel, +) error { + nodesClient := r.client.Node(model.Node.ValueString()) + storageClient := nodesClient.Storage(model.Storage.ValueString()) + + datastoresFiles, err := storageClient.ListDatastoreFiles(ctx) + if err != nil { + return fmt.Errorf("unexpected error when listing datastore files: %w", err) + } + + for _, file := range datastoresFiles { + if file != nil { + if file.VolumeID != model.ID.ValueString() { + continue + } + + model.Size = types.Int64Value(file.FileSize) + + return nil + } + } + + return fmt.Errorf("file does not exists in datastore") +} + +// Read reads file from datastore. +func (r *downloadFileResource) Read( + ctx context.Context, + req resource.ReadRequest, + resp *resource.ReadResponse, +) { + var state downloadFileModel + diags := req.State.Get(ctx, &state) + + resp.Diagnostics.Append(diags...) + + if resp.Diagnostics.HasError() { + return + } + + setOriginalValue := []byte(strconv.FormatInt(state.Size.ValueInt64(), 10)) + resp.Private.SetKey(ctx, "original_state_size", setOriginalValue) + + err := r.read(ctx, &state) + if err != nil { + if strings.Contains(err.Error(), "failed to authenticate") { + resp.Diagnostics.AddError("Failed to authenticate", err.Error()) + + return + } + + resp.Diagnostics.AddWarning( + "The file does not exist in datastore and resource must be recreated.", + err.Error(), + ) + resp.State.RemoveResource(ctx) + + return + } + + if state.Overwrite.ValueBool() { + // with overwrite, use url to get proper target size + urlMetadata, err := r.getURLMetadata( + ctx, + &state, + ) + if err != nil { + resp.Diagnostics.AddError( + "Could not get file metadata from url.", + err.Error(), + ) + + return + } + + if urlMetadata.Size != nil { + setValue := []byte(strconv.FormatInt(*urlMetadata.Size, 10)) + resp.Private.SetKey(ctx, "url_size", setValue) + } + } + + resp.Diagnostics.Append(resp.State.Set(ctx, state)...) +} + +// Update file resource. +func (r *downloadFileResource) Update( + ctx context.Context, + req resource.UpdateRequest, + resp *resource.UpdateResponse, +) { + var plan, state downloadFileModel + + resp.Diagnostics.Append(req.Plan.Get(ctx, &plan)...) + resp.Diagnostics.Append(req.State.Get(ctx, &state)...) + + err := r.read(ctx, &plan) + if err != nil { + resp.Diagnostics.AddError( + "Error when reading file from datastore", err.Error(), + ) + + return + } + + resp.Diagnostics.Append(resp.State.Set(ctx, plan)...) +} + +// Delete removes file resource. +func (r *downloadFileResource) Delete( + ctx context.Context, + req resource.DeleteRequest, + resp *resource.DeleteResponse, +) { + var state downloadFileModel + + resp.Diagnostics.Append(req.State.Get(ctx, &state)...) + + if resp.Diagnostics.HasError() { + return + } + + nodesClient := r.client.Node(state.Node.ValueString()) + storageClient := nodesClient.Storage(state.Storage.ValueString()) + + err := storageClient.DeleteDatastoreFile( + ctx, + state.ID.ValueString(), + ) + if err != nil { + if strings.Contains(err.Error(), "unable to parse") { + resp.Diagnostics.AddWarning( + "Datastore file does not exists", + fmt.Sprintf( + "Could not delete datastore file '%s', it does not exist or has been deleted outside of Terraform.", + state.ID.ValueString(), + ), + ) + } else { + resp.Diagnostics.AddError( + "Error deleting datastore file", + fmt.Sprintf("Could not delete datastore file '%s', unexpected error: %s", + state.ID.ValueString(), err.Error()), + ) + } + } +} diff --git a/fwprovider/tests/resource_download_file_test.go b/fwprovider/tests/resource_download_file_test.go new file mode 100644 index 00000000..873571ed --- /dev/null +++ b/fwprovider/tests/resource_download_file_test.go @@ -0,0 +1,134 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +package tests + +import ( + "context" + "fmt" + "testing" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" +) + +const ( + accTestDownloadIsoFileName = "proxmox_virtual_environment_download_file.iso_image" + accTestDownloadQcow2FileName = "proxmox_virtual_environment_download_file.qcow2_image" +) + +func TestAccResourceDownloadFile(t *testing.T) { + t.Parallel() + + accProviders := testAccMuxProviders(context.Background(), t) + + resource.Test(t, resource.TestCase{ + ProtoV6ProviderFactories: accProviders, + Steps: []resource.TestStep{ + // Create and Read testing + { + Config: testAccResourceDownloadIsoFileCreatedConfig(), + Check: testAccResourceDownloadIsoFileCreatedCheck(), + }, + { + Config: testAccResourceDownloadQcow2FileCreatedConfig(), + Check: testAccResourceDownloadQcow2FileCreatedCheck(), + }, + // Update testing + { + Config: testAccResourceDownloadIsoFileUpdatedConfig(), + Check: testAccResourceDownloadIsoFileUpdatedCheck(), + }, + }, + }) +} + +func testAccResourceDownloadIsoFileCreatedConfig() string { + return fmt.Sprintf(` + resource "proxmox_virtual_environment_download_file" "iso_image" { + content_type = "iso" + node_name = "%s" + datastore_id = "%s" + url = "https://cdn.githubraw.com/rafsaf/a4b19ea5e3485f8da6ca4acf46d09650/raw/d340ec3ddcef9b907ede02f64b5d3f694da5d081/fake_file.iso" + } + `, accTestNodeName, accTestStorageName) +} + +func testAccResourceDownloadQcow2FileCreatedConfig() string { + return fmt.Sprintf(` + resource "proxmox_virtual_environment_download_file" "qcow2_image" { + content_type = "iso" + node_name = "%s" + datastore_id = "%s" + file_name = "fake_qcow2_file.img" + url = "https://cdn.githubraw.com/rafsaf/036eece601975a3ad632a77fc2809046/raw/10500012fca9b4425b50de67a7258a12cba0c076/fake_file.qcow2" + checksum = "688787d8ff144c502c7f5cffaafe2cc588d86079f9de88304c26b0cb99ce91c6" + checksum_algorithm = "sha256" + } + `, accTestNodeName, accTestStorageName) +} + +func testAccResourceDownloadIsoFileCreatedCheck() resource.TestCheckFunc { + return resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr(accTestDownloadIsoFileName, "id", "local:iso/fake_file.iso"), + resource.TestCheckResourceAttr(accTestDownloadIsoFileName, "node_name", accTestNodeName), + resource.TestCheckResourceAttr(accTestDownloadIsoFileName, "datastore_id", accTestStorageName), + resource.TestCheckResourceAttr(accTestDownloadIsoFileName, "url", "https://cdn.githubraw.com/rafsaf/a4b19ea5e3485f8da6ca4acf46d09650/raw/d340ec3ddcef9b907ede02f64b5d3f694da5d081/fake_file.iso"), + resource.TestCheckResourceAttr(accTestDownloadIsoFileName, "file_name", "fake_file.iso"), + resource.TestCheckResourceAttr(accTestDownloadIsoFileName, "upload_timeout", "600"), + resource.TestCheckResourceAttr(accTestDownloadIsoFileName, "size", "3"), + resource.TestCheckResourceAttr(accTestDownloadIsoFileName, "verify", "true"), + resource.TestCheckNoResourceAttr(accTestDownloadIsoFileName, "checksum"), + resource.TestCheckNoResourceAttr(accTestDownloadIsoFileName, "checksum_algorithm"), + resource.TestCheckNoResourceAttr(accTestDownloadIsoFileName, "decompression_algorithm"), + ) +} + +func testAccResourceDownloadQcow2FileCreatedCheck() resource.TestCheckFunc { + return resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr(accTestDownloadQcow2FileName, "id", "local:iso/fake_qcow2_file.img"), + resource.TestCheckResourceAttr(accTestDownloadQcow2FileName, "content_type", "iso"), + resource.TestCheckResourceAttr(accTestDownloadQcow2FileName, "node_name", accTestNodeName), + resource.TestCheckResourceAttr(accTestDownloadQcow2FileName, "datastore_id", accTestStorageName), + resource.TestCheckResourceAttr(accTestDownloadQcow2FileName, "url", "https://cdn.githubraw.com/rafsaf/036eece601975a3ad632a77fc2809046/raw/10500012fca9b4425b50de67a7258a12cba0c076/fake_file.qcow2"), + resource.TestCheckResourceAttr(accTestDownloadQcow2FileName, "file_name", "fake_qcow2_file.img"), + resource.TestCheckResourceAttr(accTestDownloadQcow2FileName, "upload_timeout", "600"), + resource.TestCheckResourceAttr(accTestDownloadQcow2FileName, "size", "3"), + resource.TestCheckResourceAttr(accTestDownloadQcow2FileName, "verify", "true"), + resource.TestCheckResourceAttr(accTestDownloadQcow2FileName, "checksum", "688787d8ff144c502c7f5cffaafe2cc588d86079f9de88304c26b0cb99ce91c6"), + resource.TestCheckResourceAttr(accTestDownloadQcow2FileName, "checksum_algorithm", "sha256"), + resource.TestCheckNoResourceAttr(accTestDownloadQcow2FileName, "decompression_algorithm"), + ) +} + +func testAccResourceDownloadIsoFileUpdatedConfig() string { + return fmt.Sprintf(` + resource "proxmox_virtual_environment_download_file" "iso_image" { + content_type = "iso" + node_name = "%s" + datastore_id = "%s" + file_name = "fake_iso_file.img" + url = "https://cdn.githubraw.com/rafsaf/a4b19ea5e3485f8da6ca4acf46d09650/raw/d340ec3ddcef9b907ede02f64b5d3f694da5d081/fake_file.iso" + upload_timeout = 10000 + } + `, accTestNodeName, accTestStorageName) +} + +func testAccResourceDownloadIsoFileUpdatedCheck() resource.TestCheckFunc { + return resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr(accTestDownloadIsoFileName, "id", "local:iso/fake_iso_file.img"), + resource.TestCheckResourceAttr(accTestDownloadIsoFileName, "content_type", "iso"), + resource.TestCheckResourceAttr(accTestDownloadIsoFileName, "node_name", accTestNodeName), + resource.TestCheckResourceAttr(accTestDownloadIsoFileName, "datastore_id", accTestStorageName), + resource.TestCheckResourceAttr(accTestDownloadIsoFileName, "url", "https://cdn.githubraw.com/rafsaf/a4b19ea5e3485f8da6ca4acf46d09650/raw/d340ec3ddcef9b907ede02f64b5d3f694da5d081/fake_file.iso"), + resource.TestCheckResourceAttr(accTestDownloadIsoFileName, "file_name", "fake_iso_file.img"), + resource.TestCheckResourceAttr(accTestDownloadIsoFileName, "upload_timeout", "10000"), + resource.TestCheckResourceAttr(accTestDownloadIsoFileName, "size", "3"), + resource.TestCheckResourceAttr(accTestDownloadIsoFileName, "verify", "true"), + resource.TestCheckNoResourceAttr(accTestDownloadIsoFileName, "checksum"), + resource.TestCheckNoResourceAttr(accTestDownloadIsoFileName, "checksum_algorithm"), + resource.TestCheckNoResourceAttr(accTestDownloadIsoFileName, "decompression_algorithm"), + ) +} diff --git a/fwprovider/tests/resource_file_test.go b/fwprovider/tests/resource_file_test.go index 2a48e825..1bbf9369 100644 --- a/fwprovider/tests/resource_file_test.go +++ b/fwprovider/tests/resource_file_test.go @@ -173,7 +173,7 @@ func createFile(t *testing.T, namePattern string, content string) *os.File { func deleteSnippet(t *testing.T, fname string) { t.Helper() - err := getNodesClient().DeleteDatastoreFile(context.Background(), "local", fmt.Sprintf("snippets/%s", fname)) + err := getNodeStorageClient().DeleteDatastoreFile(context.Background(), fmt.Sprintf("snippets/%s", fname)) require.NoError(t, err) } diff --git a/fwprovider/tests/test_support.go b/fwprovider/tests/test_support.go index 8d0dbc4b..cf4efd20 100644 --- a/fwprovider/tests/test_support.go +++ b/fwprovider/tests/test_support.go @@ -23,12 +23,14 @@ import ( "github.com/bpg/terraform-provider-proxmox/fwprovider" "github.com/bpg/terraform-provider-proxmox/proxmox/api" "github.com/bpg/terraform-provider-proxmox/proxmox/nodes" + "github.com/bpg/terraform-provider-proxmox/proxmox/nodes/storage" sdkV2provider "github.com/bpg/terraform-provider-proxmox/proxmoxtf/provider" "github.com/bpg/terraform-provider-proxmox/utils" ) const ( - accTestNodeName = "pve" + accTestNodeName = "pve" + accTestStorageName = "local" ) // testAccMuxProviders returns a map of mux servers for the acceptance tests. @@ -106,3 +108,8 @@ func getNodesClient() *nodes.Client { return nodesClient } + +func getNodeStorageClient() *storage.Client { + nodesClient := getNodesClient() + return &storage.Client{Client: nodesClient, StorageName: accTestStorageName} +} diff --git a/proxmox/nodes/client.go b/proxmox/nodes/client.go index 6b8d6529..2e5fe5e7 100644 --- a/proxmox/nodes/client.go +++ b/proxmox/nodes/client.go @@ -12,6 +12,7 @@ import ( "github.com/bpg/terraform-provider-proxmox/proxmox/api" "github.com/bpg/terraform-provider-proxmox/proxmox/nodes/containers" + "github.com/bpg/terraform-provider-proxmox/proxmox/nodes/storage" "github.com/bpg/terraform-provider-proxmox/proxmox/nodes/tasks" "github.com/bpg/terraform-provider-proxmox/proxmox/nodes/vms" ) @@ -43,6 +44,14 @@ func (c *Client) VM(vmID int) *vms.Client { } } +// Storage returns a client for managing a specific storage. +func (c *Client) Storage(storageName string) *storage.Client { + return &storage.Client{ + Client: c, + StorageName: storageName, + } +} + // Tasks returns a client for managing VM tasks. func (c *Client) Tasks() *tasks.Client { return &tasks.Client{ diff --git a/proxmox/nodes/query_url_metadata.go b/proxmox/nodes/query_url_metadata.go new file mode 100644 index 00000000..ea5841c2 --- /dev/null +++ b/proxmox/nodes/query_url_metadata.go @@ -0,0 +1,34 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +package nodes + +import ( + "context" + "fmt" + "net/http" + + "github.com/bpg/terraform-provider-proxmox/proxmox/api" +) + +// GetQueryURLMetadata retrieves the URL filename details for a node. +func (c *Client) GetQueryURLMetadata( + ctx context.Context, + d *QueryURLMetadataGetRequestBody, +) (*QueryURLMetadataGetResponseData, error) { + resBody := &QueryURLMetadataGetResponseBody{} + + err := c.DoRequest(ctx, http.MethodGet, c.ExpandPath("query-url-metadata"), d, resBody) + if err != nil { + return nil, fmt.Errorf("error retrieving Query URL metadata configuration: %w", err) + } + + if resBody.Data == nil { + return nil, api.ErrNoDataObjectInResponse + } + + return resBody.Data, nil +} diff --git a/proxmox/nodes/query_url_metadata_types.go b/proxmox/nodes/query_url_metadata_types.go new file mode 100644 index 00000000..b234effa --- /dev/null +++ b/proxmox/nodes/query_url_metadata_types.go @@ -0,0 +1,27 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +package nodes + +import "github.com/bpg/terraform-provider-proxmox/proxmox/types" + +// QueryURLMetadataGetResponseBody contains the body from a QueryURLMetadata get response. +type QueryURLMetadataGetResponseBody struct { + Data *QueryURLMetadataGetResponseData `json:"data,omitempty"` +} + +// QueryURLMetadataGetResponseData contains the data from a QueryURLMetadata get response. +type QueryURLMetadataGetResponseData struct { + Filename *string `json:"filename,omitempty" url:"filename,omitempty"` + Mimetype *string `json:"mimetype,omitempty" url:"mimetype,omitempty"` + Size *int64 `json:"size,omitempty" url:"size,omitempty"` +} + +// QueryURLMetadataGetRequestBody contains the body for a QueryURLMetadata get request. +type QueryURLMetadataGetRequestBody struct { + Verify *types.CustomBool `json:"verify-certificates,omitempty" url:"verify-certificates,omitempty,int"` + URL *string `json:"url,omitempty" url:"url,omitempty"` +} diff --git a/proxmox/nodes/storage/client.go b/proxmox/nodes/storage/client.go new file mode 100644 index 00000000..deba9dc2 --- /dev/null +++ b/proxmox/nodes/storage/client.go @@ -0,0 +1,41 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +package storage + +import ( + "fmt" + + "github.com/bpg/terraform-provider-proxmox/proxmox/api" + "github.com/bpg/terraform-provider-proxmox/proxmox/nodes/tasks" +) + +// Client is an interface for accessing the Proxmox node storage API. +type Client struct { + api.Client + StorageName string +} + +func (c *Client) basePath() string { + return c.Client.ExpandPath("storage") +} + +// ExpandPath expands a relative path to a full node storage API path. +func (c *Client) ExpandPath(path string) string { + ep := fmt.Sprintf("%s/%s", c.basePath(), c.StorageName) + if path != "" { + ep = fmt.Sprintf("%s/%s", ep, path) + } + + return ep +} + +// Tasks returns a client for managing node storage tasks. +func (c *Client) Tasks() *tasks.Client { + return &tasks.Client{ + Client: c.Client, + } +} diff --git a/proxmox/nodes/storage/content.go b/proxmox/nodes/storage/content.go new file mode 100644 index 00000000..bb45a4df --- /dev/null +++ b/proxmox/nodes/storage/content.go @@ -0,0 +1,104 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +package storage + +import ( + "context" + "fmt" + "net/http" + "net/url" + "sort" + + "github.com/bpg/terraform-provider-proxmox/proxmox/api" +) + +// DeleteDatastoreFile deletes a file in a datastore. +func (c *Client) DeleteDatastoreFile( + ctx context.Context, + volumeID string, +) error { + err := c.DoRequest( + ctx, + http.MethodDelete, + c.ExpandPath( + fmt.Sprintf( + "content/%s", + url.PathEscape(volumeID), + ), + ), + nil, + nil, + ) + if err != nil { + return fmt.Errorf("error deleting file %s from datastore %s: %w", volumeID, c.StorageName, err) + } + + return nil +} + +// ListDatastoreFiles retrieves a list of the files in a datastore. +func (c *Client) ListDatastoreFiles( + ctx context.Context, +) ([]*DatastoreFileListResponseData, error) { + resBody := &DatastoreFileListResponseBody{} + + err := c.DoRequest( + ctx, + http.MethodGet, + c.ExpandPath("content"), + nil, + resBody, + ) + if err != nil { + return nil, fmt.Errorf("error retrieving files from datastore %s: %w", c.StorageName, err) + } + + if resBody.Data == nil { + return nil, api.ErrNoDataObjectInResponse + } + + sort.Slice(resBody.Data, func(i, j int) bool { + return resBody.Data[i].VolumeID < resBody.Data[j].VolumeID + }) + + return resBody.Data, nil +} + +// GetDatastoreFile get a file details in a datastore. +func (c *Client) GetDatastoreFile( + ctx context.Context, + volumeID string, + nodeName string, +) (*DatastoreFileGetResponseData, error) { + reqBody := &DatastoreFileGetRequestData{ + Node: nodeName, + VolumeID: volumeID, + } + resBody := &DatastoreFileGetResponseBody{} + + err := c.DoRequest( + ctx, + http.MethodGet, + c.ExpandPath( + fmt.Sprintf( + "content/%s", + url.PathEscape(volumeID), + ), + ), + reqBody, + resBody, + ) + if err != nil { + return nil, fmt.Errorf("error get file %s from datastore %s: %w", volumeID, c.StorageName, err) + } + + if resBody.Data == nil { + return nil, api.ErrNoDataObjectInResponse + } + + return resBody.Data, nil +} diff --git a/proxmox/nodes/storage/content_types.go b/proxmox/nodes/storage/content_types.go new file mode 100644 index 00000000..d1f6d326 --- /dev/null +++ b/proxmox/nodes/storage/content_types.go @@ -0,0 +1,42 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +package storage + +// DatastoreFileListResponseBody contains the body from a datastore content list response. +type DatastoreFileListResponseBody struct { + Data []*DatastoreFileListResponseData `json:"data,omitempty"` +} + +// DatastoreFileListResponseData contains the data from a datastore content list response. +type DatastoreFileListResponseData struct { + ContentType string `json:"content"` + FileFormat string `json:"format"` + FileSize int64 `json:"size"` + ParentVolumeID *string `json:"parent,omitempty"` + SpaceUsed *int `json:"used,omitempty"` + VMID *int `json:"vmid,omitempty"` + VolumeID string `json:"volid"` +} + +// DatastoreFileGetRequestData contains the body from a datastore content get request. +type DatastoreFileGetRequestData struct { + Node string `json:"node,omitempty" url:"node,omitempty"` + VolumeID string `json:"volume,omitempty" url:"volume,omitempty"` +} + +// DatastoreFileGetResponseBody contains the body from a datastore content get response. +type DatastoreFileGetResponseBody struct { + Data *DatastoreFileGetResponseData `json:"data,omitempty" url:"data,omitempty"` +} + +// DatastoreFileGetResponseData contains the data from a datastore content get response. +type DatastoreFileGetResponseData struct { + Path *string `json:"path" url:"path,omitempty"` + FileFormat *string `json:"format" url:"format,omitempty"` + FileSize *int64 `json:"size" url:"size,omitempty"` + SpaceUsed *int64 `json:"used,omitempty" url:"used,omitempty"` +} diff --git a/proxmox/nodes/storage/download_url.go b/proxmox/nodes/storage/download_url.go new file mode 100644 index 00000000..ff44c9a7 --- /dev/null +++ b/proxmox/nodes/storage/download_url.go @@ -0,0 +1,51 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +package storage + +import ( + "context" + "fmt" + "net/http" + + "github.com/bpg/terraform-provider-proxmox/proxmox/api" +) + +// DownloadFileByURL downloads the file using URL. +func (c *Client) DownloadFileByURL( + ctx context.Context, + d *DownloadURLPostRequestBody, + uploadTimeout int64, +) error { + resBody := &DownloadURLResponseBody{} + + err := c.DoRequest(ctx, http.MethodPost, c.ExpandPath("download-url"), d, resBody) + if err != nil { + return fmt.Errorf("error download file by URL: %w", err) + } + + if resBody.TaskID == nil { + return api.ErrNoDataObjectInResponse + } + + taskErr := c.Tasks().WaitForTask(ctx, *resBody.TaskID, int(uploadTimeout), 5) + if taskErr != nil { + err = fmt.Errorf( + "error download file to datastore %s: failed waiting for url download - %w", + c.StorageName, + taskErr, + ) + + deleteErr := c.Tasks().DeleteTask(context.WithoutCancel(ctx), *resBody.TaskID) + if deleteErr != nil { + return fmt.Errorf("%w \n %w", err, deleteErr) + } + + return err + } + + return nil +} diff --git a/proxmox/nodes/storage/download_url_types.go b/proxmox/nodes/storage/download_url_types.go new file mode 100644 index 00000000..1292addd --- /dev/null +++ b/proxmox/nodes/storage/download_url_types.go @@ -0,0 +1,27 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +package storage + +import "github.com/bpg/terraform-provider-proxmox/proxmox/types" + +// DownloadURLResponseBody contains the body from a DownloadURL post response. +type DownloadURLResponseBody struct { + TaskID *string `json:"data,omitempty"` +} + +// DownloadURLPostRequestBody contains the body for a DownloadURL post request. +type DownloadURLPostRequestBody struct { + Content *string `json:"content,omitempty" url:"content,omitempty"` + FileName *string `json:"filename,omitempty" url:"filename,omitempty"` + Node *string `json:"node,omitempty" url:"node,omitempty"` + Storage *string `json:"storage,omitempty" url:"storage,omitempty"` + URL *string `json:"url,omitempty" url:"url,omitempty"` + Checksum *string `json:"checksum,omitempty" url:"checksum,omitempty"` + ChecksumAlgorithm *string `json:"checksum-algorithm,omitempty" url:"checksum-algorithm,omitempty"` + Compression *string `json:"compression,omitempty" url:"compression,omitempty"` + Verify *types.CustomBool `json:"verify-certificates,omitempty" url:"verify-certificates,omitempty,int"` +} diff --git a/proxmox/nodes/storage/status.go b/proxmox/nodes/storage/status.go new file mode 100644 index 00000000..a5b41fed --- /dev/null +++ b/proxmox/nodes/storage/status.go @@ -0,0 +1,39 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +package storage + +import ( + "context" + "fmt" + "net/http" + + "github.com/bpg/terraform-provider-proxmox/proxmox/api" +) + +// GetDatastoreStatus gets status information for a given datastore. +func (c *Client) GetDatastoreStatus( + ctx context.Context, +) (*DatastoreGetStatusResponseData, error) { + resBody := &DatastoreGetStatusResponseBody{} + + err := c.DoRequest( + ctx, + http.MethodGet, + c.ExpandPath("status"), + nil, + resBody, + ) + if err != nil { + return nil, fmt.Errorf("error retrieving status for datastore %s: %w", c.StorageName, err) + } + + if resBody.Data == nil { + return nil, api.ErrNoDataObjectInResponse + } + + return resBody.Data, nil +} diff --git a/proxmox/nodes/storage/status_types.go b/proxmox/nodes/storage/status_types.go new file mode 100644 index 00000000..dc7c0690 --- /dev/null +++ b/proxmox/nodes/storage/status_types.go @@ -0,0 +1,28 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +package storage + +import ( + "github.com/bpg/terraform-provider-proxmox/proxmox/types" +) + +// DatastoreGetStatusResponseBody contains the body from a datastore status get request. +type DatastoreGetStatusResponseBody struct { + Data *DatastoreGetStatusResponseData `json:"data,omitempty"` +} + +// DatastoreGetStatusResponseData contains the data from a datastore status get request. +type DatastoreGetStatusResponseData struct { + Active *types.CustomBool `json:"active,omitempty"` + AvailableBytes *int64 `json:"avail,omitempty"` + Content *types.CustomCommaSeparatedList `json:"content,omitempty" url:"content,omitempty,comma"` + Enabled *types.CustomBool `json:"enabled,omitempty"` + Shared *types.CustomBool `json:"shared,omitempty"` + TotalBytes *int64 `json:"total,omitempty"` + Type *string `json:"type,omitempty"` + UsedBytes *int64 `json:"used,omitempty"` +} diff --git a/proxmox/nodes/storage/storage.go b/proxmox/nodes/storage/storage.go new file mode 100644 index 00000000..f414bc1e --- /dev/null +++ b/proxmox/nodes/storage/storage.go @@ -0,0 +1,45 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +package storage + +import ( + "context" + "fmt" + "net/http" + "sort" + + "github.com/bpg/terraform-provider-proxmox/proxmox/api" +) + +// ListDatastores retrieves a list of nodes. +func (c *Client) ListDatastores( + ctx context.Context, + d *DatastoreListRequestBody, +) ([]*DatastoreListResponseData, error) { + resBody := &DatastoreListResponseBody{} + + err := c.DoRequest( + ctx, + http.MethodGet, + c.basePath(), + d, + resBody, + ) + if err != nil { + return nil, fmt.Errorf("error retrieving datastores: %w", err) + } + + if resBody.Data == nil { + return nil, api.ErrNoDataObjectInResponse + } + + sort.Slice(resBody.Data, func(i, j int) bool { + return resBody.Data[i].ID < resBody.Data[j].ID + }) + + return resBody.Data, nil +} diff --git a/proxmox/nodes/storage_types.go b/proxmox/nodes/storage/storage_types.go similarity index 52% rename from proxmox/nodes/storage_types.go rename to proxmox/nodes/storage/storage_types.go index ee83f19c..fee9a153 100644 --- a/proxmox/nodes/storage_types.go +++ b/proxmox/nodes/storage/storage_types.go @@ -4,45 +4,12 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -package nodes +package storage import ( "github.com/bpg/terraform-provider-proxmox/proxmox/types" ) -// DatastoreFileListResponseBody contains the body from a datastore content list response. -type DatastoreFileListResponseBody struct { - Data []*DatastoreFileListResponseData `json:"data,omitempty"` -} - -// DatastoreFileListResponseData contains the data from a datastore content list response. -type DatastoreFileListResponseData struct { - ContentType string `json:"content"` - FileFormat string `json:"format"` - FileSize int64 `json:"size"` - ParentVolumeID *string `json:"parent,omitempty"` - SpaceUsed *int `json:"used,omitempty"` - VMID *int `json:"vmid,omitempty"` - VolumeID string `json:"volid"` -} - -// DatastoreGetStatusResponseBody contains the body from a datastore status get request. -type DatastoreGetStatusResponseBody struct { - Data *DatastoreGetStatusResponseData `json:"data,omitempty"` -} - -// DatastoreGetStatusResponseData contains the data from a datastore status get request. -type DatastoreGetStatusResponseData struct { - Active *types.CustomBool `json:"active,omitempty"` - AvailableBytes *int64 `json:"avail,omitempty"` - Content *types.CustomCommaSeparatedList `json:"content,omitempty" url:"content,omitempty,comma"` - Enabled *types.CustomBool `json:"enabled,omitempty"` - Shared *types.CustomBool `json:"shared,omitempty"` - TotalBytes *int64 `json:"total,omitempty"` - Type *string `json:"type,omitempty"` - UsedBytes *int64 `json:"used,omitempty"` -} - // DatastoreListRequestBody contains the body for a datastore list request. type DatastoreListRequestBody struct { ContentTypes types.CustomCommaSeparatedList `json:"content,omitempty" url:"content,omitempty,comma"` @@ -70,8 +37,3 @@ type DatastoreListResponseData struct { SpaceUsedPercentage *float64 `json:"used_fraction,omitempty"` Type string `json:"type,omitempty"` } - -// DatastoreUploadResponseBody contains the body from a datastore upload response. -type DatastoreUploadResponseBody struct { - UploadID *string `json:"data,omitempty"` -} diff --git a/proxmox/nodes/storage.go b/proxmox/nodes/storage/upload.go similarity index 55% rename from proxmox/nodes/storage.go rename to proxmox/nodes/storage/upload.go index b9834cdd..f2bea726 100644 --- a/proxmox/nodes/storage.go +++ b/proxmox/nodes/storage/upload.go @@ -1,10 +1,4 @@ -/* - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at https://mozilla.org/MPL/2.0/. - */ - -package nodes +package storage import ( "context" @@ -12,137 +6,16 @@ import ( "io" "mime/multipart" "net/http" - "net/url" "os" - "sort" "github.com/hashicorp/terraform-plugin-log/tflog" "github.com/bpg/terraform-provider-proxmox/proxmox/api" ) -// DeleteDatastoreFile deletes a file in a datastore. -func (c *Client) DeleteDatastoreFile( - ctx context.Context, - datastoreID, volumeID string, -) error { - err := c.DoRequest( - ctx, - http.MethodDelete, - c.ExpandPath( - fmt.Sprintf( - "storage/%s/content/%s", - url.PathEscape(datastoreID), - url.PathEscape(volumeID), - ), - ), - nil, - nil, - ) - if err != nil { - return fmt.Errorf("error deleting file %s from datastore %s: %w", volumeID, datastoreID, err) - } - - return nil -} - -// GetDatastoreStatus gets status information for a given datastore. -func (c *Client) GetDatastoreStatus( - ctx context.Context, - datastoreID string, -) (*DatastoreGetStatusResponseData, error) { - resBody := &DatastoreGetStatusResponseBody{} - - err := c.DoRequest( - ctx, - http.MethodGet, - c.ExpandPath( - fmt.Sprintf( - "storage/%s/status", - url.PathEscape(datastoreID), - ), - ), - nil, - resBody, - ) - if err != nil { - return nil, fmt.Errorf("error retrieving status for datastore %s: %w", datastoreID, err) - } - - if resBody.Data == nil { - return nil, api.ErrNoDataObjectInResponse - } - - return resBody.Data, nil -} - -// ListDatastoreFiles retrieves a list of the files in a datastore. -func (c *Client) ListDatastoreFiles( - ctx context.Context, - datastoreID string, -) ([]*DatastoreFileListResponseData, error) { - resBody := &DatastoreFileListResponseBody{} - - err := c.DoRequest( - ctx, - http.MethodGet, - c.ExpandPath( - fmt.Sprintf( - "storage/%s/content", - url.PathEscape(datastoreID), - ), - ), - nil, - resBody, - ) - if err != nil { - return nil, fmt.Errorf("error retrieving files from datastore %s: %w", datastoreID, err) - } - - if resBody.Data == nil { - return nil, api.ErrNoDataObjectInResponse - } - - sort.Slice(resBody.Data, func(i, j int) bool { - return resBody.Data[i].VolumeID < resBody.Data[j].VolumeID - }) - - return resBody.Data, nil -} - -// ListDatastores retrieves a list of nodes. -func (c *Client) ListDatastores( - ctx context.Context, - d *DatastoreListRequestBody, -) ([]*DatastoreListResponseData, error) { - resBody := &DatastoreListResponseBody{} - - err := c.DoRequest( - ctx, - http.MethodGet, - c.ExpandPath("storage"), - d, - resBody, - ) - if err != nil { - return nil, fmt.Errorf("error retrieving datastores: %w", err) - } - - if resBody.Data == nil { - return nil, api.ErrNoDataObjectInResponse - } - - sort.Slice(resBody.Data, func(i, j int) bool { - return resBody.Data[i].ID < resBody.Data[j].ID - }) - - return resBody.Data, nil -} - // APIUpload uploads a file to a datastore using the Proxmox API. func (c *Client) APIUpload( ctx context.Context, - datastoreID string, d *api.FileUploadRequest, uploadTimeout int, tempDir string, @@ -264,27 +137,22 @@ func (c *Client) APIUpload( err = c.DoRequest( ctx, http.MethodPost, - c.ExpandPath( - fmt.Sprintf( - "storage/%s/upload", - url.PathEscape(datastoreID), - ), - ), + c.ExpandPath("upload"), reqBody, resBody, ) if err != nil { - return nil, fmt.Errorf("error uploading file to datastore %s: %w", datastoreID, err) + return nil, fmt.Errorf("error uploading file to datastore %s: %w", c.StorageName, err) } if resBody.UploadID == nil { - return nil, fmt.Errorf("error uploading file to datastore %s: no uploadID", datastoreID) + return nil, fmt.Errorf("error uploading file to datastore %s: no uploadID", c.StorageName) } err = c.Tasks().WaitForTask(ctx, *resBody.UploadID, uploadTimeout, 5) if err != nil { - return nil, fmt.Errorf("error uploading file to datastore %s: failed waiting for upload - %w", datastoreID, err) + return nil, fmt.Errorf("error uploading file to datastore %s: failed waiting for upload - %w", c.StorageName, err) } return resBody, nil diff --git a/proxmox/nodes/storage/upload_types.go b/proxmox/nodes/storage/upload_types.go new file mode 100644 index 00000000..04e82c55 --- /dev/null +++ b/proxmox/nodes/storage/upload_types.go @@ -0,0 +1,12 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +package storage + +// DatastoreUploadResponseBody contains the body from a datastore upload response. +type DatastoreUploadResponseBody struct { + UploadID *string `json:"data,omitempty"` +} diff --git a/proxmox/nodes/tasks/client.go b/proxmox/nodes/tasks/client.go index d7725185..d0ae1403 100644 --- a/proxmox/nodes/tasks/client.go +++ b/proxmox/nodes/tasks/client.go @@ -23,13 +23,26 @@ func (c *Client) ExpandPath(_ string) string { panic("ExpandPath of tasks.Client must not be used. Use BuildPath instead.") } -// BuildPath builds a path using information from Task ID. -func (c *Client) BuildPath(taskID string, path string) (string, error) { +func (c *Client) baseTaskPath(taskID string) (string, error) { tid, err := ParseTaskID(taskID) if err != nil { return "", err } - return fmt.Sprintf("nodes/%s/tasks/%s/%s", - url.PathEscape(tid.NodeName), url.PathEscape(taskID), url.PathEscape(path)), nil + return fmt.Sprintf("nodes/%s/tasks/%s", + url.PathEscape(tid.NodeName), + url.PathEscape(taskID), + ), nil +} + +// BuildPath builds a path using information from Task ID. +func (c *Client) BuildPath(taskID string, path string) (string, error) { + basePath, err := c.baseTaskPath(taskID) + if err != nil { + return "", err + } + + return fmt.Sprintf("%s/%s", + basePath, url.PathEscape(path), + ), nil } diff --git a/proxmox/nodes/tasks/tasks.go b/proxmox/nodes/tasks/tasks.go index 90c88f92..521f152b 100644 --- a/proxmox/nodes/tasks/tasks.go +++ b/proxmox/nodes/tasks/tasks.go @@ -77,6 +77,27 @@ func (c *Client) GetTaskLog(ctx context.Context, upid string) ([]string, error) return lines, nil } +// DeleteTask deletes specific task. +func (c *Client) DeleteTask(ctx context.Context, upid string) error { + path, err := c.baseTaskPath(upid) + if err != nil { + return fmt.Errorf("error creating task path: %w", err) + } + + err = c.DoRequest( + ctx, + http.MethodDelete, + path, + nil, + nil, + ) + if err != nil { + return fmt.Errorf("error deleting task: %w", err) + } + + return nil +} + // WaitForTask waits for a specific task to complete. func (c *Client) WaitForTask(ctx context.Context, upid string, timeoutSec, delaySec int) error { timeDelay := int64(delaySec) diff --git a/proxmoxtf/datasource/datastores.go b/proxmoxtf/datasource/datastores.go index 1eb0a0ee..28a7574a 100644 --- a/proxmoxtf/datasource/datastores.go +++ b/proxmoxtf/datasource/datastores.go @@ -111,7 +111,7 @@ func datastoresRead(ctx context.Context, d *schema.ResourceData, m interface{}) } nodeName := d.Get(mkDataSourceVirtualEnvironmentDatastoresNodeName).(string) - list, err := api.Node(nodeName).ListDatastores(ctx, nil) + list, err := api.Node(nodeName).Storage("").ListDatastores(ctx, nil) if err != nil { return diag.FromErr(err) } diff --git a/proxmoxtf/resource/file.go b/proxmoxtf/resource/file.go index d05fbf5d..f2a01849 100644 --- a/proxmoxtf/resource/file.go +++ b/proxmoxtf/resource/file.go @@ -319,7 +319,7 @@ func fileCreate(ctx context.Context, d *schema.ResourceData, m interface{}) diag return diag.FromErr(err) } - list, err := capi.Node(nodeName).ListDatastoreFiles(ctx, datastoreID) + list, err := capi.Node(nodeName).Storage(datastoreID).ListDatastoreFiles(ctx) if err != nil { return diag.FromErr(err) } @@ -522,7 +522,7 @@ func fileCreate(ctx context.Context, d *schema.ResourceData, m interface{}) diag switch *contentType { case "iso", "vztmpl": uploadTimeout := d.Get(mkResourceVirtualEnvironmentFileTimeoutUpload).(int) - _, err = capi.Node(nodeName).APIUpload(ctx, datastoreID, request, uploadTimeout, config.TempDir()) + _, err = capi.Node(nodeName).Storage(datastoreID).APIUpload(ctx, request, uploadTimeout, config.TempDir()) default: // For all other content types, we need to upload the file to the node's // datastore using SFTP. @@ -716,7 +716,7 @@ func fileRead(ctx context.Context, d *schema.ResourceData, m interface{}) diag.D nodeName := d.Get(mkResourceVirtualEnvironmentFileNodeName).(string) sourceFile := d.Get(mkResourceVirtualEnvironmentFileSourceFile).([]interface{}) - list, err := capi.Node(nodeName).ListDatastoreFiles(ctx, datastoreID) + list, err := capi.Node(nodeName).Storage(datastoreID).ListDatastoreFiles(ctx) if err != nil { return diag.FromErr(err) } @@ -876,7 +876,7 @@ func fileDelete(ctx context.Context, d *schema.ResourceData, m interface{}) diag datastoreID := d.Get(mkResourceVirtualEnvironmentFileDatastoreID).(string) nodeName := d.Get(mkResourceVirtualEnvironmentFileNodeName).(string) - err = capi.Node(nodeName).DeleteDatastoreFile(ctx, datastoreID, d.Id()) + err = capi.Node(nodeName).Storage(datastoreID).DeleteDatastoreFile(ctx, d.Id()) if err != nil { if strings.Contains(err.Error(), "HTTP 404") { diff --git a/proxmoxtf/resource/vm.go b/proxmoxtf/resource/vm.go index 3f938237..0cfd7004 100644 --- a/proxmoxtf/resource/vm.go +++ b/proxmoxtf/resource/vm.go @@ -1871,7 +1871,7 @@ func vmCreateClone(ctx context.Context, d *schema.ResourceData, m interface{}) d onlySharedDatastores := true for _, datastore := range datastores { - datastoreStatus, err2 := api.Node(cloneNodeName).GetDatastoreStatus(ctx, datastore) + datastoreStatus, err2 := api.Node(cloneNodeName).Storage(datastore).GetDatastoreStatus(ctx) if err2 != nil { return diag.FromErr(err2) } @@ -4187,7 +4187,7 @@ func vmReadCustom( if datastoreID != "" { // disk format may not be returned by config API if it is default for the storage, and that may be different // from the default qcow2, so we need to read it from the storage API to make sure we have the correct value - files, err := api.Node(nodeName).ListDatastoreFiles(ctx, datastoreID) + files, err := api.Node(nodeName).Storage(datastoreID).ListDatastoreFiles(ctx) if err != nil { diags = append(diags, diag.FromErr(err)...) continue @@ -4292,7 +4292,7 @@ func vmReadCustom( } else { // disk format may not be returned by config API if it is default for the storage, and that may be different // from the default qcow2, so we need to read it from the storage API to make sure we have the correct value - files, err := api.Node(nodeName).ListDatastoreFiles(ctx, fileIDParts[0]) + files, err := api.Node(nodeName).Storage(fileIDParts[0]).ListDatastoreFiles(ctx) if err != nil { diags = append(diags, diag.FromErr(err)...) } else { diff --git a/tools/tools.go b/tools/tools.go index a519932e..016a4baf 100644 --- a/tools/tools.go +++ b/tools/tools.go @@ -38,3 +38,4 @@ import ( //go:generate cp ../build/docs-gen/resources/virtual_environment_hagroup.md ../docs/resources/ //go:generate cp ../build/docs-gen/resources/virtual_environment_haresource.md ../docs/resources/ //go:generate cp ../build/docs-gen/resources/virtual_environment_cluster_options.md ../docs/resources/ +//go:generate cp ../build/docs-gen/resources/virtual_environment_download_file.md ../docs/resources/