feat: use command dispatcher for pixi build (#4156)

Author: baszalmstra

Cargo.lock

@@ -301,7 +301,6 @@ pixi_allocator = { workspace = true, optional = true }
 pixi_build_discovery = { workspace = true }
 pixi_build_frontend = { workspace = true }
 pixi_build_type_conversions = { workspace = true }
-pixi_build_types = { workspace = true }
 pixi_command_dispatcher = { workspace = true }
 pixi_config = { workspace = true }
 pixi_consts = { workspace = true }

Cargo.toml

@@ -20,12 +20,12 @@ use pixi_build_types::{
         },
     },
 };
-use pixi_record::SourceRecord;
+use pixi_record::PinnedSourceSpec;
 use rattler_conda_types::{ChannelConfig, ChannelUrl, Platform, Version};
 use serde::Serialize;
 use thiserror::Error;
 
-use crate::{BuildEnvironment, CommandDispatcher, CommandDispatcherError, build::WorkDirKey};
+use crate::{BuildEnvironment, CommandDispatcherError, PackageIdentifier};
 
 /// The `BackendSourceBuildSpec` struct is used to define the specifications for
 /// building a source package using a pre-instantiated backend. This task
@@ -40,10 +40,16 @@ pub struct BackendSourceBuildSpec {
     pub backend: Backend,
 
     /// The package that we are building.
-    pub record: SourceRecord,
+    pub package: PackageIdentifier,
+
+    /// The source location of the package that we are building.
+    pub source: PinnedSourceSpec,
 
     /// The method to use for building the source package.
     pub method: BackendSourceBuildMethod,
+
+    /// The working directory to use for the build.
+    pub work_directory: PathBuf,
 }
 
 #[derive(Debug, Serialize)]
@@ -115,27 +121,28 @@ pub struct BackendBuiltSource {
 impl BackendSourceBuildSpec {
     pub async fn build(
         self,
-        command_dispatcher: CommandDispatcher,
         log_sink: UnboundedSender<String>,
     ) -> Result<BackendBuiltSource, CommandDispatcherError<BackendSourceBuildError>> {
         match self.method {
             BackendSourceBuildMethod::BuildV0(params) => {
                 Self::build_v0(
                     self.backend,
-                    self.record,
+                    self.package,
+                    self.source,
                     params,
+                    self.work_directory,
                     log_sink,
-                    command_dispatcher,
                 )
                 .await
             }
             BackendSourceBuildMethod::BuildV1(params) => {
                 Self::build_v1(
                     self.backend,
-                    self.record,
+                    self.package,
+                    self.source,
                     params,
+                    self.work_directory,
                     log_sink,
-                    command_dispatcher,
                 )
                 .await
             }
@@ -144,10 +151,11 @@ impl BackendSourceBuildSpec {
 
     async fn build_v0(
         backend: Backend,
-        record: SourceRecord,
+        record: PackageIdentifier,
+        source: PinnedSourceSpec,
         params: BackendSourceBuildV0Method,
+        work_directory: PathBuf,
         mut log_sink: UnboundedSender<String>,
-        command_dispatcher: CommandDispatcher,
     ) -> Result<BackendBuiltSource, CommandDispatcherError<BackendSourceBuildError>> {
         // Use the backend to build the source package.
         let mut build_result = backend
@@ -161,29 +169,22 @@ impl BackendSourceBuildSpec {
                         base_url: params.channel_config.channel_alias.clone(),
                     },
                     outputs: Some(BTreeSet::from_iter([CondaOutputIdentifier {
-                        name: Some(record.package_record.name.as_normalized().to_string()),
-                        version: Some(record.package_record.version.to_string()),
-                        build: Some(record.package_record.build.clone()),
-                        subdir: Some(record.package_record.subdir.clone()),
+                        name: Some(record.name.as_normalized().to_string()),
+                        version: Some(record.version.to_string()),
+                        build: Some(record.build.clone()),
+                        subdir: Some(record.subdir.clone()),
                     }])),
                     variant_configuration: params
                         .variants
                         .map(|variants| variants.into_iter().collect()),
-                    work_directory: command_dispatcher.cache_dirs().working_dirs().join(
-                        WorkDirKey {
-                            source: Box::new(record.clone()).into(),
-                            host_platform: params.build_environment.host_platform,
-                            build_backend: backend.identifier().to_string(),
-                        }
-                        .key(),
-                    ),
+                    work_directory,
                     host_platform: Some(PlatformAndVirtualPackages {
                         platform: params.build_environment.host_platform,
                         virtual_packages: Some(
                             params.build_environment.host_virtual_packages.clone(),
                         ),
                     }),
-                    editable: !record.source.is_immutable(),
+                    editable: source.is_mutable(),
                 },
                 move |line| {
                     let _err = futures::executor::block_on(log_sink.send(line));
@@ -206,8 +207,8 @@ impl BackendSourceBuildSpec {
             });
             tracing::warn!(
                 "While building {} for {}, the build backend returned more packages than expected: {pkgs}. Only the package matching the source record will be used.",
-                record.source,
-                record.package_record.subdir,
+                source,
+                record.subdir,
             );
         }
 
@@ -221,10 +222,10 @@ impl BackendSourceBuildSpec {
         } else {
             return Err(CommandDispatcherError::Failed(
                 BackendSourceBuildError::UnexpectedPackage(UnexpectedPackageError {
-                    subdir: record.package_record.subdir.clone(),
-                    name: record.package_record.name.as_normalized().to_string(),
-                    version: record.package_record.version.to_string(),
-                    build: record.package_record.build.clone(),
+                    subdir: record.subdir.clone(),
+                    name: record.name.as_normalized().to_string(),
+                    version: record.version.to_string(),
+                    build: record.build.clone(),
                     packages: build_result
                         .packages
                         .iter()
@@ -250,20 +251,12 @@ impl BackendSourceBuildSpec {
 
     async fn build_v1(
         backend: Backend,
-        record: SourceRecord,
+        record: PackageIdentifier,
+        source: PinnedSourceSpec,
         params: BackendSourceBuildV1Method,
+        work_directory: PathBuf,
         mut log_sink: UnboundedSender<String>,
-        command_dispatcher: CommandDispatcher,
     ) -> Result<BackendBuiltSource, CommandDispatcherError<BackendSourceBuildError>> {
-        let work_directory = command_dispatcher.cache_dirs().working_dirs().join(
-            WorkDirKey {
-                source: Box::new(record.clone()).into(),
-                host_platform: params.host_prefix.platform,
-                build_backend: backend.identifier().to_string(),
-            }
-            .key(),
-        );
-
         let built_package = backend
             .conda_build_v1(
                 CondaBuildV1Params {
@@ -276,19 +269,18 @@ impl BackendSourceBuildSpec {
                         platform: params.host_prefix.platform,
                     }),
                     output: CondaBuildV1Output {
-                        name: record.package_record.name.clone(),
-                        version: Some(record.package_record.version.clone()),
-                        build: Some(record.package_record.build.clone()),
+                        name: record.name.clone(),
+                        version: Some(record.version.clone()),
+                        build: Some(record.build.clone()),
                         subdir: record
-                            .package_record
                             .subdir
                             .parse()
                             .expect("found a package record with an unparsable subdir"),
                         variant: params.variant,
                     },
                     work_directory,
                     output_directory: params.output_directory,
-                    editable: Some(!record.source.is_immutable()),
+                    editable: Some(source.is_mutable()),
                 },
                 move |line| {
                     let _err = futures::executor::block_on(log_sink.send(line));
@@ -302,10 +294,10 @@ impl BackendSourceBuildSpec {
         if v1_built_package_matches_requested(&built_package, &record) {
             return Err(CommandDispatcherError::Failed(
                 BackendSourceBuildError::UnexpectedPackage(UnexpectedPackageError {
-                    subdir: record.package_record.subdir.clone(),
-                    name: record.package_record.name.as_normalized().to_string(),
-                    version: record.package_record.version.to_string(),
-                    build: record.package_record.build.clone(),
+                    subdir: record.subdir.clone(),
+                    name: record.name.as_normalized().to_string(),
+                    version: record.version.to_string(),
+                    build: record.build.clone(),
                     packages: vec![format!(
                         "{}/{}={}={}",
                         built_package.subdir,
@@ -326,23 +318,23 @@ impl BackendSourceBuildSpec {
 
 /// Returns true if the requested package matches the one that was built by a
 /// backend.
-fn v0_built_package_matches_request(record: &SourceRecord, pkg: &&CondaBuiltPackage) -> bool {
-    pkg.name == record.package_record.name
-        && Version::from_str(&pkg.version).ok().as_ref() == Some(&record.package_record.version)
-        && pkg.build == record.package_record.build
-        && pkg.subdir == record.package_record.subdir
+fn v0_built_package_matches_request(record: &PackageIdentifier, pkg: &&CondaBuiltPackage) -> bool {
+    pkg.name == record.name
+        && Version::from_str(&pkg.version).ok().as_ref() == Some(&record.version)
+        && pkg.build == record.build
+        && pkg.subdir == record.subdir
 }
 
 /// Returns true if the requested package matches the one that was built by a
 /// backend.
 fn v1_built_package_matches_requested(
     built_package: &CondaBuildV1Result,
-    record: &SourceRecord,
+    record: &PackageIdentifier,
 ) -> bool {
-    built_package.name != record.package_record.name.as_normalized()
-        || built_package.version != record.package_record.version
-        || built_package.build != record.package_record.build
-        || built_package.subdir.as_str() != record.package_record.subdir
+    built_package.name != record.name.as_normalized()
+        || built_package.version != record.version
+        || built_package.build != record.build
+        || built_package.subdir.as_str() != record.subdir
 }
 
 #[derive(Debug, thiserror::Error, Diagnostic)]

crates/pixi_command_dispatcher/src/backend_source_build/mod.rs

@@ -19,7 +19,7 @@ pub use dependencies::{Dependencies, DependenciesError, PixiRunExports};
 pub(crate) use move_file::{MoveError, move_file};
 use pixi_record::PinnedSourceSpec;
 use url::Url;
-pub use work_dir_key::WorkDirKey;
+pub use work_dir_key::{SourceRecordOrCheckout, WorkDirKey};
 use xxhash_rust::xxh3::Xxh3;
 
 const KNOWN_SUFFIXES: [&str; 3] = [".git", ".tar.gz", ".zip"];

crates/pixi_command_dispatcher/src/build/mod.rs

@@ -15,7 +15,7 @@ use serde::{Deserialize, Serialize};
 use thiserror::Error;
 use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
 
-use crate::{BuildEnvironment, build::source_checkout_cache_key};
+use crate::{BuildEnvironment, PackageIdentifier, build::source_checkout_cache_key};
 
 /// A cache for caching the metadata of a source checkout.
 ///
@@ -231,3 +231,30 @@ pub enum MetadataKind {
     /// The result of calling `conda/outputs` on a build backend.
     Outputs { outputs: Vec<CondaOutput> },
 }
+
+impl CachedCondaMetadata {
+    /// Returns the unique package identifiers for the packages in this
+    /// metadata.
+    pub fn outputs(&self) -> Vec<PackageIdentifier> {
+        match &self.metadata {
+            MetadataKind::GetMetadata { packages } => packages
+                .iter()
+                .map(|pkg| PackageIdentifier {
+                    name: pkg.name.clone(),
+                    version: pkg.version.clone(),
+                    build: pkg.build.clone(),
+                    subdir: pkg.subdir.to_string(),
+                })
+                .collect(),
+            MetadataKind::Outputs { outputs } => outputs
+                .iter()
+                .map(|output| PackageIdentifier {
+                    name: output.metadata.name.clone(),
+                    version: output.metadata.version.clone(),
+                    build: output.metadata.build.clone(),
+                    subdir: output.metadata.subdir.to_string(),
+                })
+                .collect(),
+        }
+    }
+}

crates/pixi_command_dispatcher/src/build/source_metadata_cache.rs

@@ -6,26 +6,29 @@ use std::{
 };
 
 use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD};
-use pixi_record::{PinnedSourceSpec, SourceRecord};
-use rattler_conda_types::Platform;
+use pixi_record::PinnedSourceSpec;
+use rattler_conda_types::{PackageName, Platform};
 use xxhash_rust::xxh3::Xxh3;
 
 use crate::SourceCheckout;
 
 #[derive(derive_more::From)]
 pub enum SourceRecordOrCheckout {
     /// A source record that has not been checked out yet.
-    Record(Box<SourceRecord>),
+    Record {
+        pinned: PinnedSourceSpec,
+        package_name: PackageName,
+    },
 
     /// A source checkout that has already been checked out.
-    Checkout(Box<SourceCheckout>),
+    Checkout { checkout: SourceCheckout },
 }
 
 impl SourceRecordOrCheckout {
     pub fn pinned(&self) -> &PinnedSourceSpec {
         match self {
-            SourceRecordOrCheckout::Record(record) => &record.source,
-            SourceRecordOrCheckout::Checkout(checkout) => &checkout.pinned,
+            SourceRecordOrCheckout::Record { pinned, .. } => pinned,
+            SourceRecordOrCheckout::Checkout { checkout } => &checkout.pinned,
         }
     }
 }
@@ -55,10 +58,10 @@ impl WorkDirKey {
         let unique_key = URL_SAFE_NO_PAD.encode(hasher.finish().to_ne_bytes());
 
         let name = match &self.source {
-            SourceRecordOrCheckout::Record(record) => {
-                Some(record.package_record.name.as_normalized())
+            SourceRecordOrCheckout::Record { package_name, .. } => {
+                Some(package_name.as_normalized())
             }
-            SourceRecordOrCheckout::Checkout(checkout) => {
+            SourceRecordOrCheckout::Checkout { checkout } => {
                 checkout.path.file_name().and_then(OsStr::to_str)
             }
         };