From e9e1c63c9cc21d86e63cbf8834a1b1d9c32afbd4 Mon Sep 17 00:00:00 2001 From: RunasSudo Date: Thu, 2 Sep 2021 17:17:45 +1000 Subject: [PATCH] Implement serialised binary format --- Cargo.lock | 129 ++++++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 15 +++--- src/cli/convert.rs | 18 +++++-- src/cli/stv.rs | 33 ++++++++---- src/constraints.rs | 6 +++ src/election.rs | 35 ++++++++++++ src/parser/bin.rs | 33 ++++++++++++ src/parser/mod.rs | 4 ++ src/writer/bin.rs | 35 ++++++++++++ src/writer/mod.rs | 4 ++ 10 files changed, 289 insertions(+), 23 deletions(-) create mode 100644 src/parser/bin.rs create mode 100644 src/writer/bin.rs diff --git a/Cargo.lock b/Cargo.lock index 9ef8534..a8e25d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,11 +1,24 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "adler" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "ahash" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43bb833f0bf979d8475d38fbf09ed3b8a55e1885fe93ad3f93239fc6a4f17b98" +dependencies = [ + "getrandom", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" version = "0.7.18" @@ -80,6 +93,27 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631" +[[package]] +name = "bytecheck" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb738a1e65989ecdcd5bba16079641bd7209688fa546e1064832fd6e012fd32a" +dependencies = [ + "bytecheck_derive", + "ptr_meta", +] + +[[package]] +name = "bytecheck_derive" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b4dff26fdc9f847dab475c9fec16f2cba82d5aa1f09981b87c44520721e10a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "byteorder" version = "1.4.3" @@ -329,6 +363,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "getrandom" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi", +] + [[package]] name = "git-version" version = "0.3.4" @@ -367,6 +412,15 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" +dependencies = [ + "ahash", +] + [[package]] name = "heck" version = "0.3.2" @@ -402,7 +456,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.9.1", ] [[package]] @@ -545,6 +599,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "once_cell" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" + [[package]] name = "opaque-debug" version = "0.3.0" @@ -572,6 +632,7 @@ dependencies = [ "num-traits", "paste", "predicates", + "rkyv", "rug", "sha2", "utf8-chars", @@ -668,6 +729,26 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "ptr_meta" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "quote" version = "1.0.9" @@ -724,6 +805,40 @@ version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +[[package]] +name = "rend" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0351a2e529ee30d571ef31faa5a4e0b9addaad087697b77efb20d2809e41c7" +dependencies = [ + "bytecheck", +] + +[[package]] +name = "rkyv" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e804c561b577f5836dc8a1962b7f7a03eae36f716dcd5f779c5d52a0e9c09a7" +dependencies = [ + "bytecheck", + "hashbrown 0.11.2", + "ptr_meta", + "rend", + "rkyv_derive", + "seahash", +] + +[[package]] +name = "rkyv_derive" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0afbc272334d4a4896e382508531f941a7d9505057d7424bcbed653682ce661e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "rug" version = "1.12.0" @@ -750,6 +865,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" + [[package]] name = "semver" version = "0.11.0" @@ -876,6 +997,12 @@ dependencies = [ "libc", ] +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + [[package]] name = "wasm-bindgen" version = "0.2.74" diff --git a/Cargo.toml b/Cargo.toml index c49bbec..da36119 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,20 +32,21 @@ paste = "1.0.5" assert_cmd = "1.0.5" csv = "1.1.6" flate2 = "1.0" +rkyv = "0.7.15" utf8-chars = "1.0.2" xmltree = "0.10.3" +[target.'cfg(not(target_arch = "wasm32"))'.dependencies.clap] +#version = "3.0.0-beta.4" # proc-macro2 version conflict with rkyv +git = "https://github.com/clap-rs/clap" +branch = "master" +default-features = false +features = ["std", "derive"] + [target.'cfg(not(target_arch = "wasm32"))'.dependencies.rug] version = "1.12" default-features = false features = ["integer", "rational", "float"] -[target.'cfg(not(target_arch = "wasm32"))'.dependencies.clap] -#version = "3.0.0-beta.2" # Bug 2279 -git = "https://github.com/clap-rs/clap" -branch = "master" -default-features = false -features = ["std", "derive"] - [profile.test] opt-level = 3 diff --git a/src/cli/convert.rs b/src/cli/convert.rs index f9d7a18..be2f0eb 100644 --- a/src/cli/convert.rs +++ b/src/cli/convert.rs @@ -37,11 +37,11 @@ pub struct SubcmdOptions { outfile: String, /// Format of input file - #[clap(help_heading=Some("INPUT/OUTPUT"), short, long, possible_values=&["blt", "csp"], value_name="format")] + #[clap(help_heading=Some("INPUT/OUTPUT"), short, long, possible_values=&["bin", "blt", "csp"], value_name="format")] r#in: Option, /// Format of output file - #[clap(help_heading=Some("INPUT/OUTPUT"), short, long, possible_values=&["blt", "csp"], value_name="format")] + #[clap(help_heading=Some("INPUT/OUTPUT"), short, long, possible_values=&["bin", "blt", "csp"], value_name="format")] out: Option, /// Number of seats @@ -53,7 +53,9 @@ pub struct SubcmdOptions { pub fn main(mut cmd_opts: SubcmdOptions) -> Result<(), i32> { // Auto-detect input/output formats if cmd_opts.r#in == None { - if cmd_opts.infile.ends_with(".blt") { + if cmd_opts.infile.ends_with(".bin") { + cmd_opts.r#in = Some("bin".to_string()); + } else if cmd_opts.infile.ends_with(".blt") { cmd_opts.r#in = Some("blt".to_string()); } else if cmd_opts.infile.ends_with(".csp") { cmd_opts.r#in = Some("csp".to_string()); @@ -63,7 +65,9 @@ pub fn main(mut cmd_opts: SubcmdOptions) -> Result<(), i32> { } } if cmd_opts.out == None { - if cmd_opts.outfile.ends_with(".blt") { + if cmd_opts.outfile.ends_with(".bin") { + cmd_opts.out = Some("bin".to_string()); + } else if cmd_opts.outfile.ends_with(".blt") { cmd_opts.out = Some("blt".to_string()); } else if cmd_opts.outfile.ends_with(".csp") { cmd_opts.out = Some("csp".to_string()); @@ -77,6 +81,9 @@ pub fn main(mut cmd_opts: SubcmdOptions) -> Result<(), i32> { let mut election: Election; match cmd_opts.r#in.as_deref().unwrap() { + "bin" => { + election = parser::bin::parse_path(cmd_opts.infile); + } "blt" => { match parser::blt::parse_path(cmd_opts.infile) { Ok(e) => { @@ -111,6 +118,9 @@ pub fn main(mut cmd_opts: SubcmdOptions) -> Result<(), i32> { let output = File::create(cmd_opts.outfile).expect("IO Error"); match cmd_opts.out.as_deref().unwrap() { + "bin" => { + writer::bin::write(election, output); + } "blt" => { writer::blt::write(election, output); } diff --git a/src/cli/stv.rs b/src/cli/stv.rs index 8a50f21..b9cb08b 100644 --- a/src/cli/stv.rs +++ b/src/cli/stv.rs @@ -18,7 +18,7 @@ use crate::constraints::Constraints; use crate::election::{CandidateState, CountState, Election}; use crate::numbers::{Fixed, GuardedFixed, NativeFloat64, Number, Rational}; -use crate::parser::blt; +use crate::parser::{bin, blt}; use crate::stv::{self, STVOptions}; use crate::ties; @@ -37,8 +37,13 @@ pub struct SubcmdOptions { // -- File input -- /// Path to the BLT file to be counted + #[clap(help_heading=Some("INPUT"))] filename: String, + /// Input is in serialised binary format from "opentally convert" + #[clap(help_heading=Some("INPUT"), long)] + bin: bool, + // ---------------------- // -- Numbers settings -- @@ -189,25 +194,25 @@ pub struct SubcmdOptions { pub fn main(cmd_opts: SubcmdOptions) -> Result<(), i32> { // Read and count election according to --numbers if cmd_opts.numbers == "rational" { - let mut election = election_from_file(&cmd_opts.filename)?; + let mut election = election_from_file(&cmd_opts.filename, cmd_opts.bin)?; maybe_load_constraints(&mut election, &cmd_opts.constraints); // Must specify :: here and in a few other places because ndarray causes E0275 otherwise count_election::(election, cmd_opts)?; } else if cmd_opts.numbers == "float64" { - let mut election = election_from_file(&cmd_opts.filename)?; + let mut election = election_from_file(&cmd_opts.filename, cmd_opts.bin)?; maybe_load_constraints(&mut election, &cmd_opts.constraints); count_election::(election, cmd_opts)?; } else if cmd_opts.numbers == "fixed" { Fixed::set_dps(cmd_opts.decimals); - let mut election = election_from_file(&cmd_opts.filename)?; + let mut election = election_from_file(&cmd_opts.filename, cmd_opts.bin)?; maybe_load_constraints(&mut election, &cmd_opts.constraints); count_election::(election, cmd_opts)?; } else if cmd_opts.numbers == "gfixed" { GuardedFixed::set_dps(cmd_opts.decimals); - let mut election = election_from_file(&cmd_opts.filename)?; + let mut election = election_from_file(&cmd_opts.filename, cmd_opts.bin)?; maybe_load_constraints(&mut election, &cmd_opts.constraints); count_election::(election, cmd_opts)?; } @@ -215,12 +220,18 @@ pub fn main(cmd_opts: SubcmdOptions) -> Result<(), i32> { return Ok(()); } -fn election_from_file(path: &str) -> Result, i32> { - match blt::parse_path(path) { - Ok(e) => return Ok(e), - Err(err) => { - println!("Syntax Error: {}", err); - return Err(1); +fn election_from_file(path: &str, bin: bool) -> Result, i32> { + if bin { + // BIN format + return Ok(bin::parse_path(path)); + } else { + // BLT format + match blt::parse_path(path) { + Ok(e) => return Ok(e), + Err(err) => { + println!("Syntax Error: {}", err); + return Err(1); + } } } } diff --git a/src/constraints.rs b/src/constraints.rs index c06e252..ae041bf 100644 --- a/src/constraints.rs +++ b/src/constraints.rs @@ -22,12 +22,16 @@ use crate::stv::{ConstraintMode, STVOptions}; use itertools::Itertools; use ndarray::{Array, Dimension, IxDyn}; +#[cfg(not(target_arch = "wasm32"))] +use rkyv::{Archive, Deserialize, Serialize}; + use std::collections::HashMap; use std::fmt; use std::ops; /// Constraints for an [crate::election::Election] #[derive(Debug)] +#[cfg_attr(not(target_arch = "wasm32"), derive(Archive, Deserialize, Serialize))] pub struct Constraints(pub Vec); impl Constraints { @@ -121,6 +125,7 @@ impl Constraints { /// A single dimension of constraint #[derive(Debug)] +#[cfg_attr(not(target_arch = "wasm32"), derive(Archive, Deserialize, Serialize))] pub struct Constraint { /// Name of this constraint pub name: String, @@ -130,6 +135,7 @@ pub struct Constraint { /// A group of candidates, of which a certain minimum and maximum must be elected #[derive(Debug)] +#[cfg_attr(not(target_arch = "wasm32"), derive(Archive, Deserialize, Serialize))] pub struct ConstrainedGroup { /// Name of this group pub name: String, diff --git a/src/election.rs b/src/election.rs index db93469..196679e 100644 --- a/src/election.rs +++ b/src/election.rs @@ -21,10 +21,14 @@ use crate::numbers::Number; use crate::sharandom::SHARandom; use crate::stv::{QuotaMode, STVOptions}; +#[cfg(not(target_arch = "wasm32"))] +use rkyv::{Archive, Archived, Deserialize, Fallible, Resolver, Serialize, with::{ArchiveWith, DeserializeWith, SerializeWith}}; + use std::cmp::max; use std::collections::HashMap; /// An election to be counted +#[cfg_attr(not(target_arch = "wasm32"), derive(Archive, Deserialize, Serialize))] pub struct Election { /// Name of the election pub name: String, @@ -62,6 +66,7 @@ impl Election { /// A candidate in an [Election] #[derive(PartialEq, Eq, Hash)] +#[cfg_attr(not(target_arch = "wasm32"), derive(Archive, Deserialize, Serialize))] pub struct Candidate { /// Name of the candidate pub name: String, @@ -363,13 +368,43 @@ pub struct Vote<'a, N> { } /// A record of a voter's preferences +#[cfg_attr(not(target_arch = "wasm32"), derive(Archive, Deserialize, Serialize))] pub struct Ballot { /// Original value/weight of the ballot + #[cfg_attr(not(target_arch = "wasm32"), with(SerializedNum))] pub orig_value: N, /// Indexes of candidates preferenced on the ballot pub preferences: Vec, } +/// rkyv-serialized representation of [Number] +#[cfg(not(target_arch = "wasm32"))] +pub struct SerializedNum; + +#[cfg(not(target_arch = "wasm32"))] +impl ArchiveWith for SerializedNum { + type Archived = Archived; + type Resolver = Resolver; + + unsafe fn resolve_with(field: &N, pos: usize, resolver: Self::Resolver, out: *mut Self::Archived) { + field.to_string().resolve(pos, resolver, out); + } +} + +#[cfg(not(target_arch = "wasm32"))] +impl SerializeWith for SerializedNum where String: Serialize { + fn serialize_with(field: &N, serializer: &mut S) -> Result { + return field.to_string().serialize(serializer); + } +} + +#[cfg(not(target_arch = "wasm32"))] +impl DeserializeWith, N, D> for SerializedNum where Archived: Deserialize { + fn deserialize_with(field: &Archived, deserializer: &mut D) -> Result { + return Ok(N::parse(&field.deserialize(deserializer)?)); + } +} + /// State of a [Candidate] during a count #[allow(dead_code)] #[derive(PartialEq)] diff --git a/src/parser/bin.rs b/src/parser/bin.rs new file mode 100644 index 0000000..02705ed --- /dev/null +++ b/src/parser/bin.rs @@ -0,0 +1,33 @@ +/* OpenTally: Open-source election vote counting + * Copyright © 2021 Lee Yingtong Li (RunasSudo) + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +use crate::election::Election; +use crate::numbers::Number; + +use rkyv::{Deserialize, Infallible, archived_root}; + +use std::fs; +use std::path::Path; + +/// Parse the given BIN file +pub fn parse_path, N: Number>(path: P) -> Election { + let content = fs::read(path).expect("IO Error"); + let archived = unsafe { + archived_root::>(&content) + }; + return archived.deserialize(&mut Infallible).unwrap(); +} diff --git a/src/parser/mod.rs b/src/parser/mod.rs index c48aeaa..23d4c2b 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -15,6 +15,10 @@ * along with this program. If not, see . */ +/// BIN file parser +#[cfg(not(target_arch = "wasm32"))] +pub mod bin; + /// BLT file parser pub mod blt; /// CSP file parser diff --git a/src/writer/bin.rs b/src/writer/bin.rs new file mode 100644 index 0000000..cc54bfc --- /dev/null +++ b/src/writer/bin.rs @@ -0,0 +1,35 @@ +/* OpenTally: Open-source election vote counting + * Copyright © 2021 Lee Yingtong Li (RunasSudo) + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +use crate::election::Election; +use crate::numbers::Number; + +use rkyv::ser::{Serializer, serializers::AllocSerializer}; + +use std::io::{BufWriter, Write}; + +/// Write the [Election] into BIN format +pub fn write(election: Election, output: W) { + // Serialize data using rkyv + let mut serializer = AllocSerializer::<256>::default(); + serializer.serialize_value(&election).unwrap(); + let buffer = serializer.into_serializer().into_inner(); + + // Write output + let mut output = BufWriter::new(output); + output.write(&buffer).expect("IO Error"); +} diff --git a/src/writer/mod.rs b/src/writer/mod.rs index 044a14f..2e465bf 100644 --- a/src/writer/mod.rs +++ b/src/writer/mod.rs @@ -15,6 +15,10 @@ * along with this program. If not, see . */ +/// BIN file writer +#[cfg(not(target_arch = "wasm32"))] +pub mod bin; + /// BLT file writer pub mod blt; /// CSP file writer