Refactor public parameters based on Arecibo changes (#1085)

* refactor: Refactor public parameters handling and caching

This adapts to upstream argumentcomputer/arecibo#285, which removes the need for separate `ProverKey` and `VerifierKey` instances (the former containing `CommitmentKey` copies) in Public Parameters.

- Implemented structural changes in `nova.rs` for improvements in key handling for the prover and verifier keys, which generated as `OnceCell`,

- Revamped `disk_cache.rs` with added `Arc` imports, elimination of lifetime parameter `'a` from `DiskCache`, modifications of `read` and `write` functions, and updates in the serialization and deserialization steps.
- Altered `public_params` and `supernova_public_params` functions in `mod.rs` to deal more capably with errors and modified various methods and tests to align with the changes in `disk_cache.rs`.
- Removed complete `mem_cache.rs` file, which was unsafe, and unused.

- Moved initialization of `store` to an earlier point in `fibonacci_prove` function and made minor formatting changes in `fibonacci.rs`.

* refactor: Refactor code to remove lifetime on Instance + Abomonation dependency

- Removed `Abomonation` trait and `PrimeField` imports from `multiframe.rs`, also simplified the `NonUniformCircuit` access in the same file.
- Removed `Abomonation` trait bounds from the `circuit_cache_key` functions in `supernova.rs`.
- Refactored `Instance<>` struct in `instance.rs`, removing the usage of `Abomonation` and `PhantomData`, and eliminated need for lifetime declaration `'a`.
- Enhanced `open()` method in `instance.rs` to include metadata comparison to prevent instance mismatches.

* refactor: Adapt SuperNova parameters and remove superfluous Abomonation bounds

- Improved `supernova.rs` by using `OnceCell` to store `ProverKey` and `VerifierKey` and updating related methods,
- removed uneeded Abomonation bounds in a few places.

* chore: slience clippy

* refactor: Refactor PublicParams cache to remove obsolete std::sync::Arc usage

- reduced complexity by removing `std::sync::Arc` in `src/public_parameters/disk_cache.rs`.
- Modified `read` and `write` functions in `DiskCache` to work directly with `PublicParams`.

* chore: remove anymap dependency

* chore: point the dependency back to Arecibo's dev

.clippy.toml

@@ -1,4 +1,4 @@
-type-complexity-threshold = 999
+type-complexity-threshold = 1200
 too-many-arguments-threshold = 20
 disallowed-methods = [
     # we use strict naming for pasta fields

Cargo.toml

@@ -12,7 +12,6 @@ rust-version = "1.72" # allows msrv verify to work in CI
 [dependencies]
 ahash = "0.8.6"
 anyhow = { workspace = true }
-anymap = "1.0.0-beta.2"
 base32ct = { version = "0.2.0", features = ["std"] }
 base-x = "0.2.11"
 bellpepper = { workspace = true }
@@ -125,7 +124,7 @@ clap = "4.3.17"
 ff = "0.13"
 metrics = "0.22.0"
 neptune = { git = "https://github.com/lurk-lab/neptune", branch = "dev", features = ["abomonation"] }
-nova = { git = "https://github.com/lurk-lab/arecibo", branch = "dev", package = "arecibo" }
+nova = { git = "https://github.com/lurk-lab/arecibo", branch = "dev", package = "arecibo", features = ["abomonate"] }
 once_cell = "1.18.0"
 pairing = { version = "0.23" }
 pasta_curves = { git = "https://github.com/lurk-lab/pasta_curves", branch = "dev" }

benches/fibonacci.rs

@@ -90,6 +90,7 @@ fn fibonacci_prove<M: measurement::Measurement>(
         true,
         Kind::NovaPublicParams,
     );
+    let store = Store::default();
     let pp = public_params(&instance).unwrap();
 
     // Track the number of `Lurk frames / sec`
@@ -103,8 +104,6 @@ fn fibonacci_prove<M: measurement::Measurement>(
         BenchmarkId::new(name, params),
         &prove_params,
         |b, prove_params| {
-            let store = Store::default();
-
             let ptr = fib_expr::<pasta_curves::Fq>(&store);
             let prover = NovaProver::new(prove_params.reduction_count, lang_rc.clone());
 
@@ -119,7 +118,7 @@ fn fibonacci_prove<M: measurement::Measurement>(
                     let _ = black_box(result);
                 },
                 BatchSize::LargeInput,
-            )
+            );
         },
     );
 }

benches/sha256.rs

@@ -111,7 +111,7 @@ fn sha256_ivc_prove<M: measurement::Measurement>(
     let lurk_step = make_eval_step_from_config(&EvalConfig::new_ivc(&lang));
 
     // use cached public params
-    let instance: Instance<'_, Fr, Sha256Coproc<Fr>> = Instance::new(
+    let instance: Instance<Fr, Sha256Coproc<Fr>> = Instance::new(
         reduction_count,
         lang_rc.clone(),
         true,

examples/sha256_ivc.rs

@@ -77,7 +77,7 @@ fn main() {
 
     let pp_start = Instant::now();
     let instance = Instance::new(REDUCTION_COUNT, lang_rc, true, Kind::NovaPublicParams);
-    // see the documentation on `with_public_params`
+    // see the documentation on `public_params`
     let pp = public_params(&instance).unwrap();
     let pp_end = pp_start.elapsed();
     println!("Public parameters took {:?}", pp_end);

src/cli/lurk_proof.rs

@@ -126,10 +126,7 @@ pub(crate) enum LurkProof<
     'a,
     F: CurveCycleEquipped,
     C: Coprocessor<F> + Serialize + DeserializeOwned,
-> where
-    <<E1<F> as Engine>::Scalar as ff::PrimeField>::Repr: Abomonation,
-    <<E2<F> as Engine>::Scalar as ff::PrimeField>::Repr: Abomonation,
-{
+> {
     Nova {
         proof: nova::Proof<F, C1LEM<'a, F, C>>,
         public_inputs: Vec<F>,
@@ -141,9 +138,6 @@ pub(crate) enum LurkProof<
 
 impl<'a, F: CurveCycleEquipped, C: Coprocessor<F> + 'a + Serialize + DeserializeOwned>
     HasFieldModulus for LurkProof<'a, F, C>
-where
-    <<E1<F> as Engine>::Scalar as ff::PrimeField>::Repr: Abomonation,
-    <<E2<F> as Engine>::Scalar as ff::PrimeField>::Repr: Abomonation,
 {
     fn field_modulus() -> String {
         F::MODULUS.to_owned()
@@ -152,16 +146,24 @@ where
 
 impl<'a, F: CurveCycleEquipped + Serialize, C: Coprocessor<F> + Serialize + DeserializeOwned>
     LurkProof<'a, F, C>
-where
-    <<E1<F> as Engine>::Scalar as ff::PrimeField>::Repr: Abomonation,
-    <<E2<F> as Engine>::Scalar as ff::PrimeField>::Repr: Abomonation,
 {
     #[inline]
     pub(crate) fn persist(self, proof_key: &str) -> Result<()> {
         dump(self, &proof_path(proof_key))
     }
 }
 
+impl<
+        F: CurveCycleEquipped + DeserializeOwned,
+        C: Coprocessor<F> + Serialize + DeserializeOwned + 'static,
+    > LurkProof<'static, F, C>
+{
+    #[inline]
+    pub(crate) fn is_cached(proof_key: &str) -> bool {
+        load::<Self>(&proof_path(proof_key)).is_ok()
+    }
+}
+
 impl<
         F: CurveCycleEquipped + DeserializeOwned,
         C: Coprocessor<F> + Serialize + DeserializeOwned + 'static,
@@ -180,11 +182,6 @@ where
         Ok(())
     }
 
-    #[inline]
-    pub(crate) fn is_cached(proof_key: &str) -> bool {
-        load::<Self>(&proof_path(proof_key)).is_ok()
-    }
-
     fn verify(&self) -> Result<bool> {
         match self {
             Self::Nova {

src/cli/repl/meta_cmd.rs

@@ -2,7 +2,6 @@ use ::nova::traits::Engine;
 use abomonation::Abomonation;
 use anyhow::{anyhow, bail, Context, Result};
 use camino::{Utf8Path, Utf8PathBuf};
-use ff::PrimeField;
 use serde::{de::DeserializeOwned, Deserialize, Serialize};
 use std::{collections::HashMap, process};
 
@@ -49,8 +48,8 @@ impl<
         C: Coprocessor<F> + Serialize + DeserializeOwned + 'static,
     > MetaCmd<F, C>
 where
-    <F as PrimeField>::Repr: Abomonation,
-    <<<F as CurveCycleEquipped>::E2 as Engine>::Scalar as PrimeField>::Repr: Abomonation,
+    <<E1<F> as Engine>::Scalar as ff::PrimeField>::Repr: Abomonation,
+    <<E2<F> as Engine>::Scalar as ff::PrimeField>::Repr: Abomonation,
 {
     const LOAD: MetaCmd<F, C> = MetaCmd {
         name: "load",

src/cli/repl/mod.rs

@@ -3,7 +3,6 @@ mod meta_cmd;
 use abomonation::Abomonation;
 use anyhow::{anyhow, bail, Context, Result};
 use camino::{Utf8Path, Utf8PathBuf};
-use ff::PrimeField;
 use nova::traits::Engine;
 use rustyline::{
     error::ReadlineError,
@@ -40,7 +39,7 @@ use crate::{
     },
     parser,
     proof::{
-        nova::{CurveCycleEquipped, NovaProver},
+        nova::{CurveCycleEquipped, NovaProver, E1, E2},
         RecursiveSNARKTrait,
     },
     public_parameters::{
@@ -167,8 +166,8 @@ impl<
         C: Coprocessor<F> + Serialize + DeserializeOwned + 'static,
     > Repl<F, C>
 where
-    <F as PrimeField>::Repr: Abomonation,
-    <<<F as CurveCycleEquipped>::E2 as Engine>::Scalar as PrimeField>::Repr: Abomonation,
+    <<E1<F> as Engine>::Scalar as ff::PrimeField>::Repr: Abomonation,
+    <<E2<F> as Engine>::Scalar as ff::PrimeField>::Repr: Abomonation,
 {
     pub(crate) fn new(
         store: Store<F>,
@@ -337,7 +336,6 @@ where
                     let instance =
                         Instance::new(self.rc, self.lang.clone(), true, Kind::NovaPublicParams);
                     let pp = public_params(&instance)?;
-
                     let prover = NovaProver::<_, C>::new(self.rc, self.lang.clone());
 
                     info!("Proving");

src/lem/multiframe.rs

@@ -1,10 +1,8 @@
-use abomonation::Abomonation;
 use anyhow::Result;
 use bellpepper::util_cs::witness_cs::WitnessCS;
 use bellpepper_core::{num::AllocatedNum, Circuit, ConstraintSystem, SynthesisError};
 use elsa::sync::FrozenMap;
-use ff::PrimeField;
-use nova::{supernova::NonUniformCircuit, traits::Engine};
+use nova::supernova::NonUniformCircuit;
 use once_cell::sync::OnceCell;
 use rayon::prelude::*;
 use std::sync::Arc;
@@ -904,8 +902,6 @@ impl<'a, F, C> NonUniformCircuit<E1<F>, E2<F>, MultiFrame<'a, F, C>, C2<F>> for
 where
     F: CurveCycleEquipped + LurkField,
     C: Coprocessor<F> + 'a,
-    <<E1<F> as Engine>::Scalar as PrimeField>::Repr: Abomonation,
-    <<E2<F> as Engine>::Scalar as PrimeField>::Repr: Abomonation,
 {
     fn num_circuits(&self) -> usize {
         assert_eq!(self.pc, 0);

src/proof/nova.rs

@@ -1,6 +1,4 @@
-use abomonation::Abomonation;
 use bellpepper_core::{num::AllocatedNum, ConstraintSystem};
-use ff::PrimeField;
 use halo2curves::bn256::Fr as Bn256Scalar;
 use nova::{
     errors::NovaError,
@@ -13,6 +11,7 @@ use nova::{
     },
     CompressedSNARK, ProverKey, R1CSWithArity, RecursiveSNARK, VerifierKey,
 };
+use once_cell::sync::OnceCell;
 use pasta_curves::pallas;
 use serde::{Deserialize, Serialize};
 use std::{
@@ -112,50 +111,61 @@ pub type NovaPublicParams<F, C1> = nova::PublicParams<E1<F>, E2<F>, C1, C2<F>>;
 pub struct PublicParams<F, SC: StepCircuit<F>>
 where
     F: CurveCycleEquipped,
-    // technical bounds that would disappear once associated_type_bounds stabilizes
-    <<E1<F> as Engine>::Scalar as ff::PrimeField>::Repr: Abomonation,
-    <<E2<F> as Engine>::Scalar as ff::PrimeField>::Repr: Abomonation,
 {
-    pp: NovaPublicParams<F, SC>,
-    pk: ProverKey<E1<F>, E2<F>, SC, C2<F>, SS1<F>, SS2<F>>,
-    vk: VerifierKey<E1<F>, E2<F>, SC, C2<F>, SS1<F>, SS2<F>>,
+    /// Public parameters for the Nova proving system.
+    pub pp: NovaPublicParams<F, SC>,
+    /// Prover and verifier key for final proof compression
+    #[serde(skip)]
+    pk_and_vk: OnceCell<(
+        ProverKey<E1<F>, E2<F>, SC, C2<F>, SS1<F>, SS2<F>>,
+        VerifierKey<E1<F>, E2<F>, SC, C2<F>, SS1<F>, SS2<F>>,
+    )>,
 }
 
-impl<F: CurveCycleEquipped, SC: StepCircuit<F>> Abomonation for PublicParams<F, SC>
-where
-    <<E1<F> as Engine>::Scalar as ff::PrimeField>::Repr: Abomonation,
-    <<E2<F> as Engine>::Scalar as ff::PrimeField>::Repr: Abomonation,
+// this avoids dipping into the pk/vk
+impl<F: CurveCycleEquipped, SC: StepCircuit<F> + std::fmt::Debug> std::fmt::Debug
+    for PublicParams<F, SC>
 {
-    unsafe fn entomb<W: std::io::Write>(&self, bytes: &mut W) -> std::io::Result<()> {
-        self.pp.entomb(bytes)?;
-        self.pk.entomb(bytes)?;
-        self.vk.entomb(bytes)?;
-        Ok(())
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("PublicParams")
+            .field("pp", &self.pp)
+            .finish()
+    }
+}
+
+impl<F: CurveCycleEquipped, SC: StepCircuit<F>> PublicParams<F, SC> {
+    /// provides a reference to a ProverKey suitable for producing a CompressedProof
+    pub fn pk(&self) -> &ProverKey<E1<F>, E2<F>, SC, C2<F>, SS1<F>, SS2<F>> {
+        let (pk, _vk) = self.pk_and_vk.get_or_init(|| {
+            CompressedSNARK::<E1<F>, E2<F>, SC, C2<F>, SS1<F>, SS2<F>>::setup(&self.pp).unwrap()
+        });
+        pk
     }
 
-    unsafe fn exhume<'b>(&mut self, mut bytes: &'b mut [u8]) -> Option<&'b mut [u8]> {
-        let temp = bytes;
-        bytes = self.pp.exhume(temp)?;
-        let temp = bytes;
-        bytes = self.pk.exhume(temp)?;
-        let temp = bytes;
-        bytes = self.vk.exhume(temp)?;
-        Some(bytes)
+    /// provides a reference to a VerifierKey suitable for verifying a CompressedProof
+    pub fn vk(&self) -> &VerifierKey<E1<F>, E2<F>, SC, C2<F>, SS1<F>, SS2<F>> {
+        let (_pk, vk) = self.pk_and_vk.get_or_init(|| {
+            CompressedSNARK::<E1<F>, E2<F>, SC, C2<F>, SS1<F>, SS2<F>>::setup(&self.pp).unwrap()
+        });
+        vk
     }
+}
 
-    fn extent(&self) -> usize {
-        self.pp.extent() + self.pk.extent() + self.vk.extent()
+impl<F: CurveCycleEquipped, SC: StepCircuit<F>> From<NovaPublicParams<F, SC>>
+    for PublicParams<F, SC>
+{
+    fn from(pp: NovaPublicParams<F, SC>) -> PublicParams<F, SC> {
+        PublicParams {
+            pp,
+            pk_and_vk: OnceCell::new(),
+        }
     }
 }
 
 /// An enum representing the two types of proofs that can be generated and verified.
 #[derive(Serialize, Deserialize)]
 #[serde(bound = "")]
-pub enum Proof<F: CurveCycleEquipped, C1: StepCircuit<F>>
-where
-    <<E1<F> as Engine>::Scalar as ff::PrimeField>::Repr: Abomonation,
-    <<E2<F> as Engine>::Scalar as ff::PrimeField>::Repr: Abomonation,
-{
+pub enum Proof<F: CurveCycleEquipped, C1: StepCircuit<F>> {
     /// A proof for the intermediate steps of a recursive computation along with
     /// the number of steps used for verification
     Recursive(Box<RecursiveSNARK<E1<F>, E2<F>, C1, C2<F>>>, usize),
@@ -186,11 +196,7 @@ pub fn circuit_cache_key<'a, F: CurveCycleEquipped, C: Coprocessor<F> + 'a>(
 pub fn public_params<'a, F: CurveCycleEquipped, C: Coprocessor<F> + 'a>(
     reduction_count: usize,
     lang: Arc<Lang<F, C>>,
-) -> PublicParams<F, C1LEM<'a, F, C>>
-where
-    <<E1<F> as Engine>::Scalar as ff::PrimeField>::Repr: Abomonation,
-    <<E2<F> as Engine>::Scalar as ff::PrimeField>::Repr: Abomonation,
-{
+) -> PublicParams<F, C1LEM<'a, F, C>> {
     let (circuit_primary, circuit_secondary) = circuits(reduction_count, lang);
 
     let commitment_size_hint1 = <SS1<F> as RelaxedR1CSSNARKTrait<E1<F>>>::ck_floor();
@@ -202,8 +208,10 @@ where
         &*commitment_size_hint1,
         &*commitment_size_hint2,
     );
-    let (pk, vk) = CompressedSNARK::setup(&pp).unwrap();
-    PublicParams { pp, pk, vk }
+    PublicParams {
+        pp,
+        pk_and_vk: OnceCell::new(),
+    }
 }
 
 /// Generates the circuits for the Nova proving system.
@@ -220,9 +228,6 @@ pub fn circuits<'a, F: CurveCycleEquipped, C: Coprocessor<F> + 'a>(
 
 impl<'a, F: CurveCycleEquipped, C: Coprocessor<F>> RecursiveSNARKTrait<F, C1LEM<'a, F, C>>
     for Proof<F, C1LEM<'a, F, C>>
-where
-    <F as PrimeField>::Repr: Abomonation,
-    <<<F as CurveCycleEquipped>::E2 as Engine>::Scalar as PrimeField>::Repr: Abomonation,
 {
     type PublicParams = PublicParams<F, C1LEM<'a, F, C>>;
 
@@ -342,7 +347,7 @@ where
             Self::Recursive(recursive_snark, num_steps) => Ok(Self::Compressed(
                 Box::new(CompressedSNARK::<_, _, _, _, SS1<F>, SS2<F>>::prove(
                     &pp.pp,
-                    &pp.pk,
+                    pp.pk(),
                     &recursive_snark,
                 )?),
                 num_steps,
@@ -361,7 +366,7 @@ where
                 p.verify(&pp.pp, *num_steps, z0_primary, &z0_secondary)?
             }
             Self::Compressed(p, num_steps) => {
-                p.verify(&pp.vk, *num_steps, z0_primary, &z0_secondary)?
+                p.verify(pp.vk(), *num_steps, z0_primary, &z0_secondary)?
             }
         };
 
@@ -379,11 +384,7 @@ pub struct NovaProver<'a, F: CurveCycleEquipped, C: Coprocessor<F>> {
     _phantom: PhantomData<&'a ()>,
 }
 
-impl<'a, F: CurveCycleEquipped, C: Coprocessor<F> + 'a> NovaProver<'a, F, C>
-where
-    <<E1<F> as Engine>::Scalar as ff::PrimeField>::Repr: Abomonation,
-    <<E2<F> as Engine>::Scalar as ff::PrimeField>::Repr: Abomonation,
-{
+impl<'a, F: CurveCycleEquipped, C: Coprocessor<F> + 'a> NovaProver<'a, F, C> {
     /// Create a new NovaProver with a reduction count and a `Lang`
     #[inline]
     pub fn new(reduction_count: usize, lang: Arc<Lang<F, C>>) -> Self {
@@ -417,9 +418,6 @@ where
 
 impl<'a, F: CurveCycleEquipped, C: Coprocessor<F> + 'a> Prover<'a, F, C1LEM<'a, F, C>>
     for NovaProver<'a, F, C>
-where
-    <<E1<F> as Engine>::Scalar as ff::PrimeField>::Repr: Abomonation,
-    <<E2<F> as Engine>::Scalar as ff::PrimeField>::Repr: Abomonation,
 {
     type PublicParams = PublicParams<F, C1LEM<'a, F, C>>;
     type RecursiveSnark = Proof<F, C1LEM<'a, F, C>>;