path	# lines of code
front/admin_ui/app.py	672
front/admin_ui/poetry.toml	3
front/admin_ui/pyproject.toml	16
jobs/mongodb_migration/poetry.toml	3
jobs/mongodb_migration/pyproject.toml	36
jobs/mongodb_migration/src/mongodb_migration/renaming_migrations.py	93
jobs/mongodb_migration/src/mongodb_migration/migration.py	47
jobs/mongodb_migration/src/mongodb_migration/config.py	31
jobs/mongodb_migration/src/mongodb_migration/check.py	34
jobs/mongodb_migration/src/mongodb_migration/database_migrations.py	26
jobs/mongodb_migration/src/mongodb_migration/drop_migrations.py	20
jobs/mongodb_migration/src/mongodb_migration/migrations/_20230703110100_cache_add_partial_field_in_config_parquet_and_info.py	56
jobs/mongodb_migration/src/mongodb_migration/migrations/_20240619124500_cache_add_estimated_dataset_info_field_parquet_and_info.py	42
jobs/mongodb_migration/src/mongodb_migration/migrations/_20240626151600_cache_remove_has_fts_field_in_split_duckdb_index.py	44
jobs/mongodb_migration/src/mongodb_migration/migrations/_20240112164500_cache_add_partial_field_in_split_descriptive_statistics.py	80
jobs/mongodb_migration/src/mongodb_migration/migrations/_20230516101600_queue_delete_index_without_revision.py	31
jobs/mongodb_migration/src/mongodb_migration/migrations/_20230511100700_queue_delete_indexes_with_force.py	32
jobs/mongodb_migration/src/mongodb_migration/migrations/_20240624144000_cache_add_estimated_num_rows_field_in_size.py	71
jobs/mongodb_migration/src/mongodb_migration/migrations/_20230825170200_lock_add_ttl.py	18
jobs/mongodb_migration/src/mongodb_migration/migrations/_20240703160300_cache_add_duration.py	18
jobs/mongodb_migration/src/mongodb_migration/migrations/_20240109160700_cache_add_failed_runs.py	26
jobs/mongodb_migration/src/mongodb_migration/migrations/_20240221160700_cache_merge_split_first_rows.py	63
jobs/mongodb_migration/src/mongodb_migration/migrations/_20240221160800_cache_set_updated_at_to_root_step.py	18
jobs/mongodb_migration/src/mongodb_migration/migrations/_20230705160600_queue_job_add_difficulty.py	32
jobs/mongodb_migration/src/mongodb_migration/migrations/_20240206153000_cache_add_tags_in_hub_cache.py	23
jobs/mongodb_migration/src/mongodb_migration/migrations/_20230622131500_lock_add_owner.py	21
jobs/mongodb_migration/src/mongodb_migration/migrations/_20230516101500_queue_job_add_revision.py	18
jobs/mongodb_migration/src/mongodb_migration/migrations/_20221110230400_example.py	9
jobs/mongodb_migration/src/mongodb_migration/migrations/_20221117223000_cache_generic_response.py	82
jobs/mongodb_migration/src/mongodb_migration/migrations/_20230309123100_cache_add_progress.py	18
jobs/mongodb_migration/src/mongodb_migration/migrations/__init__.py	1
jobs/mongodb_migration/src/mongodb_migration/migrations/_20221116133500_queue_job_add_force.py	15
jobs/mongodb_migration/src/mongodb_migration/migrations/_20230126164900_queue_job_add_priority.py	18
jobs/mongodb_migration/src/mongodb_migration/migrations/_20230926095900_cache_add_has_fts_field_in_split_duckdb_index.py	34
jobs/mongodb_migration/src/mongodb_migration/migrations/_20240626095000_cache_add_stemmer_in_split_duckdb_index.py	53
jobs/mongodb_migration/src/mongodb_migration/migrations/_20230824154900_cache_add_features_field_in_split_duckdb_index.py	34
jobs/mongodb_migration/src/mongodb_migration/migrations/_20240104085000_cache_add_retries.py	18
jobs/mongodb_migration/src/mongodb_migration/migrations/_20231106193200_cache_add_partial_field_in_split_duckdb_index.py	48
jobs/mongodb_migration/src/mongodb_migration/migrations/_20240221103200_cache_merge_config_split_names.py	62
jobs/mongodb_migration/src/mongodb_migration/migrations/_20230309141600_cache_add_job_runner_version.py	34
jobs/mongodb_migration/src/mongodb_migration/migrations/_20240731143600_queue_add_dataset_status_to_queue_metrics.py	21
jobs/mongodb_migration/src/mongodb_migration/deletion_migrations.py	155
jobs/mongodb_migration/src/mongodb_migration/main.py	46
jobs/mongodb_migration/src/mongodb_migration/__init__.py	1
jobs/mongodb_migration/src/mongodb_migration/collector.py	414
jobs/mongodb_migration/src/mongodb_migration/resources.py	6
jobs/mongodb_migration/src/mongodb_migration/constants.py	2
jobs/mongodb_migration/src/mongodb_migration/plan.py	77
jobs/cache_maintenance/poetry.toml	3
jobs/cache_maintenance/pyproject.toml	42
jobs/cache_maintenance/src/cache_maintenance/config.py	78
jobs/cache_maintenance/src/cache_maintenance/backfill.py	115
jobs/cache_maintenance/src/cache_maintenance/queue_metrics.py	33
jobs/cache_maintenance/src/cache_maintenance/main.py	104
jobs/cache_maintenance/src/cache_maintenance/__init__.py	1
jobs/cache_maintenance/src/cache_maintenance/cache_metrics.py	22
jobs/cache_maintenance/src/cache_maintenance/discussions.py	118
chart/Chart.yaml	15
chart/env/prod.yaml	515
chart/env/staging.yaml	290
chart/values.yaml	454
chart/templates/worker/hpa.yaml	3
chart/templates/worker/_deployment.yaml	37
chart/templates/worker/_hpa.yaml	44
chart/templates/worker/_container.tpl	58
chart/templates/worker/deployment.yaml	3
chart/templates/worker/_podmonitor.yaml	20
chart/templates/worker/podmonitor.yaml	3
chart/templates/jobs/mongodb-migration/_container.tpl	18
chart/templates/jobs/mongodb-migration/job.yaml	26
chart/templates/_volumeMounts/_volumeMountParquetMetadata.tpl	16
chart/templates/_initContainers/_initContainerParquetMetadata.tpl	20
chart/templates/secrets.yaml	24
chart/templates/services/webhook/ingress-internal.yaml	23
chart/templates/services/webhook/_container.tpl	55
chart/templates/services/webhook/deployment.yaml	31
chart/templates/services/webhook/ingress.yaml	23
chart/templates/services/webhook/service.yaml	19
chart/templates/services/webhook/pdb.yaml	10
chart/templates/services/webhook/servicemonitor.yaml	17
chart/templates/services/api/ingress-internal.yaml	41
chart/templates/services/api/_container.tpl	54
chart/templates/services/api/deployment.yaml	31
chart/templates/services/api/ingress.yaml	41
chart/templates/services/api/service.yaml	19
chart/templates/services/api/pdb.yaml	10
chart/templates/services/api/servicemonitor.yaml	17
chart/templates/services/sse-api/ingress-internal.yaml	23
chart/templates/services/sse-api/_container.tpl	42
chart/templates/services/sse-api/deployment.yaml	31
chart/templates/services/sse-api/ingress.yaml	23
chart/templates/services/sse-api/service.yaml	19
chart/templates/services/sse-api/pdb.yaml	10
chart/templates/services/sse-api/servicemonitor.yaml	17
chart/templates/services/rows/ingress-internal.yaml	23
chart/templates/services/rows/_container.tpl	58
chart/templates/services/rows/deployment.yaml	35
chart/templates/services/rows/ingress.yaml	23
chart/templates/services/rows/service.yaml	19
chart/templates/services/rows/pdb.yaml	10
chart/templates/services/rows/servicemonitor.yaml	17
chart/templates/services/admin/ingress-internal.yaml	32
chart/templates/services/admin/_container.tpl	62
chart/templates/services/admin/deployment.yaml	35
chart/templates/services/admin/ingress.yaml	32
chart/templates/services/admin/service.yaml	19
chart/templates/services/admin/pdb.yaml	10
chart/templates/services/admin/servicemonitor.yaml	17
chart/templates/services/search/ingress-internal.yaml	33
chart/templates/services/search/_container.tpl	68
chart/templates/services/search/deployment.yaml	35
chart/templates/services/search/ingress.yaml	33
chart/templates/services/search/service.yaml	19
chart/templates/services/search/pdb.yaml	10
chart/templates/services/search/servicemonitor.yaml	17
chart/templates/cron-jobs/backfill/_container.tpl	21
chart/templates/cron-jobs/backfill/job.yaml	22
chart/templates/cron-jobs/cache-metrics-collector/_container.tpl	17
chart/templates/cron-jobs/cache-metrics-collector/job.yaml	22
chart/templates/cron-jobs/backfill-retryable-errors/_container.tpl	21
chart/templates/cron-jobs/backfill-retryable-errors/job.yaml	22
chart/templates/cron-jobs/queue-metrics-collector/_container.tpl	17
chart/templates/cron-jobs/queue-metrics-collector/job.yaml	22
chart/templates/cron-jobs/post-messages/_container.tpl	19
chart/templates/cron-jobs/post-messages/job.yaml	22
chart/templates/_env/_envCache.tpl	8
chart/templates/_env/_envDiscussions.tpl	19
chart/templates/_env/_envNumba.tpl	9
chart/templates/_env/_envCommitter.tpl	14
chart/templates/_env/_envWorker.tpl	79
chart/templates/_env/_envCloudfront.tpl	26
chart/templates/_env/_envQueue.tpl	8
chart/templates/_env/_envParquetMetadata.tpl	6
chart/templates/_env/_envAssets.tpl	10
chart/templates/_env/_envCommon.tpl	35
chart/templates/_env/_envDatasetsBased.tpl	10
chart/templates/_env/_envCachedAssets.tpl	10
chart/templates/_env/_envLog.tpl	6
chart/templates/_env/_envHf.tpl	32
chart/templates/_env/_envS3.tpl	26
chart/templates/_volumes/_volumeParquetMetadata.tpl	7
chart/templates/_common/_helpers.tpl	185
chart/templates/_common/_ingressTls.tpl	16
chart/templates/_common/_dnsConfig.tpl	8
chart/templates/_common/_securityContext.tpl	7
services/worker/poetry.toml	3
services/worker/pyproject.toml	86
services/worker/src/worker/executor.py	191
services/worker/src/worker/job_runner_factory.py	215
services/worker/src/worker/dtos.py	248
services/worker/src/worker/config.py	297
services/worker/src/worker/job_runners/dataset/is_valid.py	62
services/worker/src/worker/job_runners/dataset/hub_cache.py	114
services/worker/src/worker/job_runners/dataset/dataset_job_runner.py	41
services/worker/src/worker/job_runners/dataset/parquet.py	80
services/worker/src/worker/job_runners/dataset/split_names.py	75
services/worker/src/worker/job_runners/dataset/opt_in_out_urls_count.py	74
services/worker/src/worker/job_runners/dataset/modalities.py	264
services/worker/src/worker/job_runners/dataset/compatible_libraries.py	646
services/worker/src/worker/job_runners/dataset/config_names.py	81
services/worker/src/worker/job_runners/dataset/presidio_entities_count.py	86
services/worker/src/worker/job_runners/dataset/__init__.py	1
services/worker/src/worker/job_runners/dataset/filetypes.py	80
services/worker/src/worker/job_runners/dataset/info.py	62
services/worker/src/worker/job_runners/dataset/size.py	113
services/worker/src/worker/job_runners/dataset/croissant_crumbs.py	198
services/worker/src/worker/job_runners/config/is_valid.py	62
services/worker/src/worker/job_runners/config/parquet_and_info.py	1039
services/worker/src/worker/job_runners/config/parquet.py	29
services/worker/src/worker/job_runners/config/split_names.py	91
services/worker/src/worker/job_runners/config/opt_in_out_urls_count.py	67
services/worker/src/worker/job_runners/config/config_job_runner.py	40
services/worker/src/worker/job_runners/config/parquet_metadata.py	123
services/worker/src/worker/job_runners/config/__init__.py	1
services/worker/src/worker/job_runners/config/info.py	29
services/worker/src/worker/job_runners/config/size.py	82
services/worker/src/worker/job_runners/_job_runner_with_datasets_cache.py	44
services/worker/src/worker/job_runners/__init__.py	1
services/worker/src/worker/job_runners/_job_runner_with_cache.py	49
services/worker/src/worker/job_runners/split/is_valid.py	68
services/worker/src/worker/job_runners/split/descriptive_statistics.py	251
services/worker/src/worker/job_runners/split/split_job_runner.py	59
services/worker/src/worker/job_runners/split/opt_in_out_urls_count.py	40
services/worker/src/worker/job_runners/split/presidio_scan.py	579
services/worker/src/worker/job_runners/split/opt_in_out_urls_scan_from_streaming.py	208
services/worker/src/worker/job_runners/split/__init__.py	1
services/worker/src/worker/job_runners/split/first_rows.py	225
services/worker/src/worker/job_runners/split/image_url_columns.py	72
services/worker/src/worker/start_web_app.py	22
services/worker/src/worker/utils.py	233
services/worker/src/worker/start_worker_loop.py	59
services/worker/src/worker/loop.py	98
services/worker/src/worker/routes/__init__.py	1
services/worker/src/worker/routes/healthcheck.py	6
services/worker/src/worker/routes/metrics.py	16
services/worker/src/worker/job_runner.py	23
services/worker/src/worker/main.py	59
services/worker/src/worker/__init__.py	2
services/worker/src/worker/job_manager.py	206
services/worker/src/worker/resources.py	31
services/webhook/poetry.toml	3
services/webhook/pyproject.toml	57
services/webhook/src/webhook/app.py	87
services/webhook/src/webhook/config.py	40
services/webhook/src/webhook/routes/__init__.py	1
services/webhook/src/webhook/routes/webhook.py	182
services/webhook/src/webhook/main.py	3
services/webhook/src/webhook/__init__.py	1
services/reverse-proxy/nginx-templates/404.html	5
services/api/poetry.toml	3
services/api/pyproject.toml	58
services/api/src/api/app.py	104
services/api/src/api/config.py	81
services/api/src/api/routes/endpoint.py	187
services/api/src/api/routes/__init__.py	1
services/api/src/api/main.py	3
services/api/src/api/__init__.py	1
services/sse-api/poetry.toml	3
services/sse-api/pyproject.toml	61
services/sse-api/src/sse_api/app.py	66
services/sse-api/src/sse_api/config.py	25
services/sse-api/src/sse_api/watcher.py	140
services/sse-api/src/sse_api/routes/hub_cache.py	44
services/sse-api/src/sse_api/main.py	3
services/sse-api/src/sse_api/__init__.py	1
services/sse-api/src/sse_api/constants.py	1
services/rows/poetry.toml	3
services/rows/pyproject.toml	59
services/rows/src/rows/app.py	105
services/rows/src/rows/config.py	43
services/rows/src/rows/routes/rows.py	139
services/rows/src/rows/routes/__init__.py	1
services/rows/src/rows/main.py	3
services/rows/src/rows/__init__.py	1
services/admin/poetry.toml	3
services/admin/pyproject.toml	58
services/admin/src/admin/app.py	184
services/admin/src/admin/config.py	90
services/admin/src/admin/authentication.py	68
services/admin/src/admin/routes/force_refresh.py	104
services/admin/src/admin/routes/pending_jobs.py	37
services/admin/src/admin/routes/recreate_dataset.py	74
services/admin/src/admin/routes/blocked_datasets.py	32
services/admin/src/admin/routes/__init__.py	1
services/admin/src/admin/routes/healthcheck.py	6
services/admin/src/admin/routes/cache_reports_with_content.py	51
services/admin/src/admin/routes/cache_reports.py	47
services/admin/src/admin/routes/num_dataset_infos_by_builder_name.py	77
services/admin/src/admin/routes/metrics.py	25
services/admin/src/admin/routes/dataset_status.py	47
services/admin/src/admin/main.py	3
services/admin/src/admin/__init__.py	1
services/search/poetry.toml	3
services/search/pyproject.toml	55
services/search/src/search/duckdb_connection.py	15
services/search/src/search/app.py	133
services/search/src/search/config.py	69
services/search/src/search/routes/search.py	225
services/search/src/search/routes/filter.py	195
services/search/src/search/routes/__init__.py	1
services/search/src/search/main.py	3
services/search/src/search/__init__.py	1
libs/libapi/poetry.toml	3
libs/libapi/pyproject.toml	55
libs/libapi/src/libapi/exceptions.py	106
libs/libapi/src/libapi/config.py	59
libs/libapi/src/libapi/jwt_token.py	166
libs/libapi/src/libapi/utils.py	181
libs/libapi/src/libapi/rows_utils.py	68
libs/libapi/src/libapi/authentication.py	94
libs/libapi/src/libapi/request.py	28
libs/libapi/src/libapi/response.py	52
libs/libapi/src/libapi/routes/__init__.py	1
libs/libapi/src/libapi/routes/healthcheck.py	8
libs/libapi/src/libapi/routes/metrics.py	12
libs/libapi/src/libapi/__init__.py	1
libs/libapi/src/libapi/duckdb.py	254
libs/libcommon/poetry.toml	3
libs/libcommon/pyproject.toml	103
libs/libcommon/src/libcommon/parquet_utils.py	482
libs/libcommon/src/libcommon/state.py	301
libs/libcommon/src/libcommon/simple_cache.py	620
libs/libcommon/src/libcommon/url_preparator.py	122
libs/libcommon/src/libcommon/queue/jobs.py	573
libs/libcommon/src/libcommon/queue/utils.py	12
libs/libcommon/src/libcommon/queue/dataset_blockages.py	44
libs/libcommon/src/libcommon/queue/lock.py	125
libs/libcommon/src/libcommon/queue/__init__.py	1
libs/libcommon/src/libcommon/queue/past_jobs.py	53
libs/libcommon/src/libcommon/queue/metrics.py	87
libs/libcommon/src/libcommon/viewer_utils/rows.py	129
libs/libcommon/src/libcommon/viewer_utils/asset.py	202
libs/libcommon/src/libcommon/viewer_utils/parquet_metadata.py	33
libs/libcommon/src/libcommon/viewer_utils/truncate_rows.py	67
libs/libcommon/src/libcommon/viewer_utils/__init__.py	1
libs/libcommon/src/libcommon/viewer_utils/features.py	476
libs/libcommon/src/libcommon/statistics_utils.py	677
libs/libcommon/src/libcommon/orchestrator.py	926
libs/libcommon/src/libcommon/dtos.py	93
libs/libcommon/src/libcommon/exceptions.py	357
libs/libcommon/src/libcommon/config.py	176
libs/libcommon/src/libcommon/cloudfront.py	44
libs/libcommon/src/libcommon/utils.py	205
libs/libcommon/src/libcommon/prometheus.py	107
libs/libcommon/src/libcommon/log.py	4
libs/libcommon/src/libcommon/operations.py	232
libs/libcommon/src/libcommon/croissant_utils.py	138
libs/libcommon/src/libcommon/__init__.py	2
libs/libcommon/src/libcommon/resources.py	67
libs/libcommon/src/libcommon/storage.py	70
libs/libcommon/src/libcommon/processing_graph.py	469
libs/libcommon/src/libcommon/constants.py	93
libs/libcommon/src/libcommon/storage_client.py	102