diff --git a/.gitea/workflows/benchmark.yaml b/.gitea/workflows/benchmark.yaml index 5d4c02f..17c573c 100644 --- a/.gitea/workflows/benchmark.yaml +++ b/.gitea/workflows/benchmark.yaml @@ -32,9 +32,10 @@ jobs: uv run -- pytest "${PYTEST_ARGS[@]}" - name: Upload benchmark JSON report - uses: https://git.pvv.ntnu.no/Projects/rsync-action@v1 + uses: https://git.pvv.ntnu.no/Projects/rsync-action@v2 with: source: ./benchmark/*/*.json + quote-source: false target: ${{ gitea.ref_name }}/benchmark/${{ github.run_id }}/benchmark.json username: gitea-web ssh-key: ${{ secrets.WEB_SYNC_SSH_KEY }} diff --git a/tests/benchmark/benchmark_settings.py b/tests/benchmark/benchmark_settings.py new file mode 100644 index 0000000..ca13694 --- /dev/null +++ b/tests/benchmark/benchmark_settings.py @@ -0,0 +1,11 @@ +TRANSACTION_GENERATOR_EXCEPTION_LIMIT = 15 +""" +The random transaction generator uses a set seed to generate transactions. +However, not all transactions are valid in all contexts. We catch illegal +generated transactions with a try/except, and retry until we generate a valid +one. However, if we exceed this limit, something is likely wrong with the generator +instead, due to the unlikely high number of exceptions. +""" + +BENCHMARK_ITERATIONS = 5 +BENCHMARK_ROUNDS = 3 diff --git a/tests/benchmark/helpers.py b/tests/benchmark/helpers.py index bc8240f..e3250d6 100644 --- a/tests/benchmark/helpers.py +++ b/tests/benchmark/helpers.py @@ -5,6 +5,7 @@ from sqlalchemy.orm import Session from dibbler.models import Product, Transaction, TransactionType, User from dibbler.queries import joint_buy_product +from tests.benchmark.benchmark_settings import TRANSACTION_GENERATOR_EXCEPTION_LIMIT def insert_users_and_products( @@ -66,9 +67,6 @@ def generate_random_transactions( return transactions -EXCEPTION_LIMIT = 15 - - def random_add_product_transaction(sql_session: Session, last_time: datetime) -> Transaction: i = 0 while True: @@ -89,7 +87,7 @@ def random_add_product_transaction(sql_session: Session, last_time: datetime) -> time=new_datetime, ) except Exception: - if i > EXCEPTION_LIMIT: + if i > TRANSACTION_GENERATOR_EXCEPTION_LIMIT: raise RuntimeError( "Too many failed attempts to create a valid transaction, consider changing the seed" ) @@ -111,7 +109,7 @@ def random_adjust_balance_transaction(sql_session: Session, last_time: datetime) time=new_datetime, ) except Exception: - if i > EXCEPTION_LIMIT: + if i > TRANSACTION_GENERATOR_EXCEPTION_LIMIT: raise RuntimeError( "Too many failed attempts to create a valid transaction, consider changing the seed" ) @@ -133,7 +131,7 @@ def random_adjust_interest_transaction(sql_session: Session, last_time: datetime time=new_datetime, ) except Exception: - if i > EXCEPTION_LIMIT: + if i > TRANSACTION_GENERATOR_EXCEPTION_LIMIT: raise RuntimeError( "Too many failed attempts to create a valid transaction, consider changing the seed" ) @@ -157,7 +155,7 @@ def random_adjust_penalty_transaction(sql_session: Session, last_time: datetime) time=new_datetime, ) except Exception: - if i > EXCEPTION_LIMIT: + if i > TRANSACTION_GENERATOR_EXCEPTION_LIMIT: raise RuntimeError( "Too many failed attempts to create a valid transaction, consider changing the seed" ) @@ -181,7 +179,7 @@ def random_adjust_stock_transaction(sql_session: Session, last_time: datetime) - time=new_datetime, ) except Exception: - if i > EXCEPTION_LIMIT: + if i > TRANSACTION_GENERATOR_EXCEPTION_LIMIT: raise RuntimeError( "Too many failed attempts to create a valid transaction, consider changing the seed" ) @@ -205,7 +203,7 @@ def random_buy_product_transaction(sql_session: Session, last_time: datetime) -> time=new_datetime, ) except Exception: - if i > EXCEPTION_LIMIT: + if i > TRANSACTION_GENERATOR_EXCEPTION_LIMIT: raise RuntimeError( "Too many failed attempts to create a valid transaction, consider changing the seed" ) @@ -233,7 +231,7 @@ def random_joint_transaction(sql_session: Session, last_time: datetime) -> list[ time=new_datetime, ) except Exception: - if i > EXCEPTION_LIMIT: + if i > TRANSACTION_GENERATOR_EXCEPTION_LIMIT: raise RuntimeError( "Too many failed attempts to create a valid transaction, consider changing the seed" ) @@ -256,7 +254,7 @@ def random_transfer_transaction(sql_session: Session, last_time: datetime) -> Tr time=new_datetime, ) except Exception: - if i > EXCEPTION_LIMIT: + if i > TRANSACTION_GENERATOR_EXCEPTION_LIMIT: raise RuntimeError( "Too many failed attempts to create a valid transaction, consider changing the seed" ) @@ -280,7 +278,7 @@ def random_throw_product_transaction(sql_session: Session, last_time: datetime) time=new_datetime, ) except Exception: - if i > EXCEPTION_LIMIT: + if i > TRANSACTION_GENERATOR_EXCEPTION_LIMIT: raise RuntimeError( "Too many failed attempts to create a valid transaction, consider changing the seed" ) diff --git a/tests/benchmark/test_benchmark_product_owners.py b/tests/benchmark/test_benchmark_product_owners.py index 5fcaa97..9ef5b6a 100644 --- a/tests/benchmark/test_benchmark_product_owners.py +++ b/tests/benchmark/test_benchmark_product_owners.py @@ -3,6 +3,7 @@ from sqlalchemy.orm import Session from dibbler.models import Product, TransactionType from dibbler.queries import product_owners +from tests.benchmark.benchmark_settings import BENCHMARK_ITERATIONS, BENCHMARK_ROUNDS from tests.benchmark.helpers import generate_random_transactions, insert_users_and_products @@ -36,8 +37,8 @@ def test_benchmark_product_owners(benchmark, sql_session: Session, transaction_c benchmark.pedantic( query_all_product_owners, args=(sql_session, products), - iterations=10, - rounds=5, + iterations=BENCHMARK_ITERATIONS, + rounds=BENCHMARK_ROUNDS, ) diff --git a/tests/benchmark/test_benchmark_product_price.py b/tests/benchmark/test_benchmark_product_price.py index b5a8e9c..aad31ef 100644 --- a/tests/benchmark/test_benchmark_product_price.py +++ b/tests/benchmark/test_benchmark_product_price.py @@ -3,6 +3,7 @@ from sqlalchemy.orm import Session from dibbler.models import Product, TransactionType from dibbler.queries import product_price +from tests.benchmark.benchmark_settings import BENCHMARK_ITERATIONS, BENCHMARK_ROUNDS from tests.benchmark.helpers import generate_random_transactions, insert_users_and_products @@ -33,8 +34,8 @@ def test_benchmark_product_price(benchmark, sql_session: Session, transaction_co benchmark.pedantic( query_all_products_price, args=(sql_session, products), - iterations=10, - rounds=5, + iterations=BENCHMARK_ITERATIONS, + rounds=BENCHMARK_ROUNDS, ) diff --git a/tests/benchmark/test_benchmark_product_stock.py b/tests/benchmark/test_benchmark_product_stock.py index 003c8a8..a72319c 100644 --- a/tests/benchmark/test_benchmark_product_stock.py +++ b/tests/benchmark/test_benchmark_product_stock.py @@ -3,6 +3,7 @@ from sqlalchemy.orm import Session from dibbler.models import Product, TransactionType from dibbler.queries import product_stock +from tests.benchmark.benchmark_settings import BENCHMARK_ITERATIONS, BENCHMARK_ROUNDS from tests.benchmark.helpers import generate_random_transactions, insert_users_and_products @@ -36,8 +37,8 @@ def test_benchmark_product_stock(benchmark, sql_session: Session, transaction_co benchmark.pedantic( query_all_products_stock, args=(sql_session, products), - iterations=10, - rounds=5, + iterations=BENCHMARK_ITERATIONS, + rounds=BENCHMARK_ROUNDS, ) diff --git a/tests/benchmark/test_benchmark_transaction_log.py b/tests/benchmark/test_benchmark_transaction_log.py index 25ae11d..a467049 100644 --- a/tests/benchmark/test_benchmark_transaction_log.py +++ b/tests/benchmark/test_benchmark_transaction_log.py @@ -3,6 +3,7 @@ from sqlalchemy.orm import Session from dibbler.models import Product, User from dibbler.queries import transaction_log +from tests.benchmark.benchmark_settings import BENCHMARK_ITERATIONS, BENCHMARK_ROUNDS from tests.benchmark.helpers import generate_random_transactions, insert_users_and_products @@ -33,8 +34,8 @@ def test_benchmark_transaction_log(benchmark, sql_session: Session, transaction_ products, users, ), - iterations=10, - rounds=5, + iterations=BENCHMARK_ITERATIONS, + rounds=BENCHMARK_ROUNDS, ) diff --git a/tests/benchmark/test_benchmark_user_balance.py b/tests/benchmark/test_benchmark_user_balance.py index 978c5d3..a89ec96 100644 --- a/tests/benchmark/test_benchmark_user_balance.py +++ b/tests/benchmark/test_benchmark_user_balance.py @@ -3,6 +3,7 @@ from sqlalchemy.orm import Session from dibbler.models import User from dibbler.queries import user_balance +from tests.benchmark.benchmark_settings import BENCHMARK_ITERATIONS, BENCHMARK_ROUNDS from tests.benchmark.helpers import generate_random_transactions, insert_users_and_products @@ -28,8 +29,8 @@ def test_benchmark_user_balance(benchmark, sql_session: Session, transaction_cou benchmark.pedantic( query_all_users_balance, args=(sql_session, users), - iterations=10, - rounds=5, + iterations=BENCHMARK_ITERATIONS, + rounds=BENCHMARK_ROUNDS, )