{"schema_version":"2","scorer_version":"0.1.0","scoring_version":"prov-4+qual-2+sec-2","generated_at":"2026-11-13T23:22:51.080430Z","repo":{"ref":{"host":"github.com","owner":"gnemoug","name":"distribute_crawler"},"default_branch":"master","clone_url":"https://github.com/gnemoug/distribute_crawler.git","html_url":"https://github.com/gnemoug/distribute_crawler","description":"使用scrapy,redis, mongodb,graphite实现的一个分布式网络爬虫,底层存储mongodb集群,分布式使用redis实现,爬虫状态显示使用graphite实现","stars":3234,"forks":1550,"is_fork":false,"is_archived":false,"is_private":false,"size_kb":12514,"primary_language":"Python","license_spdx":null,"created_at":"2013-04-17T09:14:02Z","pushed_at":"2017-04-18T08:00:51Z","security_policy_url":null,"distribution":{"owner_created_at":"2012-06-28T11:41:30Z","owner_type":"User","release_assets":[]}},"commit_sha":"fb74ad853126b3a6358f987d763d492d8f3eec12","profile":{"lines_by_language":{"Python":2634},"files_by_category":{"binary":946,"source":32,"docs":1,"other":1,"config":1,"test":1},"source_lines":2604,"test_lines":30},"pillars":{"ai_provenance":{"score":0.0,"confidence":0.44,"coverage":1.0,"label":"none-detected","breakdown":{"attributed_share":0.0,"suspected_share":0.0,"estimated_share":0.0,"ai_artifacts":0.0,"flagged_work_units":0.0,"flagged_releases":null},"summary":"No AI attribution or AI-consistent velocity signals detected."},"quality":{"score":45.5,"confidence":0.9,"coverage":1.0,"label":"poor","breakdown":{"complexity":100.0,"duplication":31.1,"lint":17.8,"testing":30.6,"docs":40.0,"type_safety":40.0,"process":null},"summary":"Code quality is poor: test-to-source ratio 0.01, no CI test run detected; 0% of function code in complex or long functions; 19% duplicated lines. Weakest area: lint."},"security":{"score":53.4,"confidence":0.85,"coverage":1.0,"label":"weak","breakdown":{"secrets":100.0,"sast":21.8,"dependencies":null,"ci_actions":null,"posture":0.0,"scorecard":null},"summary":"Security posture is weak: 8 high-severity SAST findings. Weakest area: posture."}},"analyzer_results":[{"analyzer":"sec-scorecard","pillar":"security","status":"skipped","tool_version":null,"metrics":{},"findings":[],"evidence":[],"data":{},"error":"no published OpenSSF Scorecard result for this repo"},{"analyzer":"sec-registry","pillar":"security","status":"skipped","tool_version":null,"metrics":{},"findings":[],"evidence":[],"data":{},"error":"no registry dependencies declared"},{"analyzer":"ai-attribution","pillar":"ai_provenance","status":"ok","tool_version":"signatures-v1","metrics":{"commits":15.0,"attributed_commits":0.0,"attributed_commit_share":0.0,"effective_loc":2637.0,"attributed_loc":0.0,"attributed_loc_share":0.0,"tools_detected":0.0},"findings":[],"evidence":[],"data":{"attributed_loc_by_commit":{},"tools":{},"monthly":{"2013-04":{"loc":2636,"attributed_loc":0},"2013-08":{"loc":1,"attributed_loc":0}}},"error":null},{"analyzer":"ai-artifacts","pillar":"ai_provenance","status":"ok","tool_version":"signatures-v1","metrics":{"artifacts":0.0,"workflow_agents":0.0,"tools_configured":0.0},"findings":[],"evidence":[],"data":{"artifacts":[],"workflows":[],"tools":[]},"error":null},{"analyzer":"ai-velocity","pillar":"ai_provenance","status":"ok","tool_version":"prov-4","metrics":{"work_units":3.0,"flagged_units":0.0,"pre_ai_era_bursts":1.0,"pre_ai_era_measured_loc":2637.0,"pre_ai_era_burst_loc":2529.0,"measured_loc":2637.0,"flagged_loc":0.0,"flagged_loc_share":0.0,"max_loc_per_hour":558.3466208757512,"median_loc_per_hour":292.5483104378756,"pr_timed_units":0.0,"excluded_import_loc":0.0},"findings":[],"evidence":[],"data":{"flagged_loc_by_commit":{},"excluded_import_commits":[]},"error":null},{"analyzer":"ai-release-velocity","pillar":"ai_provenance","status":"skipped","tool_version":null,"metrics":{},"findings":[],"evidence":[],"data":{},"error":null},{"analyzer":"ai-stylometry","pillar":"ai_provenance","status":"ok","tool_version":null,"metrics":{"lines_scanned":2604.0,"index":0.0,"count:narrated_steps":0.0,"per_kloc:narrated_steps":0.0,"files:narrated_steps":0.0,"count:placeholder":0.0,"per_kloc:placeholder":0.0,"files:placeholder":0.0,"count:emoji_in_code":0.0,"per_kloc:emoji_in_code":0.0,"files:emoji_in_code":0.0,"count:assistant_voice":0.0,"per_kloc:assistant_voice":0.0,"files:assistant_voice":0.0},"findings":[],"evidence":[],"data":{},"error":null},{"analyzer":"sec-secrets","pillar":"security","status":"ok","tool_version":"gitleaks 8.30.1","metrics":{"secrets_current":0.0,"secrets_history":0.0,"secrets_nonprod":0.0,"secrets_low_confidence":0.0},"findings":[],"evidence":[],"data":{},"error":null},{"analyzer":"sec-sast","pillar":"security","status":"ok","tool_version":"opengrep 1.33.1; rules f1d2b562b414","metrics":{"findings":14.0,"findings_high":8.0,"findings_medium":6.0,"findings_low":0.0,"weighted_findings":11.35,"scanned_lines":2604.0,"parse_errors":0.0},"findings":[{"rule_id":"opengrep/python.lang.security.audit.dangerous-system-call-audit","severity":"high","message":"Found dynamic content used in a system call. This is dangerous if external data can reach this function call because it allows a malicious actor to execute commands. Use the 'subprocess' module instead, which is easier to use without accidentally exposing a command injection vulnerability. [confidence low]","path":"woaidu_crawler/woaidu_crawler/commands/compression.py","line":21,"fingerprint":"python.lang.security.audit.dangerous-system-call-audit:woaidu_crawler/woaidu_crawler/commands/compression.py:21"},{"rule_id":"opengrep/python.lang.security.audit.dangerous-system-call-audit","severity":"high","message":"Found dynamic content used in a system call. This is dangerous if external data can reach this function call because it allows a malicious actor to execute commands. Use the 'subprocess' module instead, which is easier to use without accidentally exposing a command injection vulnerability. [confidence low]","path":"woaidu_crawler/woaidu_crawler/commands/decomperssion.py","line":33,"fingerprint":"python.lang.security.audit.dangerous-system-call-audit:woaidu_crawler/woaidu_crawler/commands/decomperssion.py:33"},{"rule_id":"opengrep/python.lang.security.audit.dangerous-subprocess-use-audit","severity":"high","message":"Detected subprocess function 'Popen' without a static string. If this data can be controlled by a malicious actor, it may be an instance of command injection. Audit the use of this call to ensure it is not controllable by an external resource. You may consider using 'shlex.escape()'. [confidence low]","path":"woaidu_crawler/woaidu_crawler/commands/init_sharding_mongodb.py","line":272,"fingerprint":"python.lang.security.audit.dangerous-subprocess-use-audit:woaidu_crawler/woaidu_crawler/commands/init_sharding_mongodb.py:272"},{"rule_id":"opengrep/python.lang.security.audit.dangerous-subprocess-use-tainted-env-args","severity":"high","message":"Detected subprocess function 'Popen' with user controlled data. A malicious actor could leverage this to perform command injection. You may consider using 'shlex.quote()'. [confidence medium]","path":"woaidu_crawler/woaidu_crawler/commands/init_sharding_mongodb.py","line":272,"fingerprint":"python.lang.security.audit.dangerous-subprocess-use-tainted-env-args:woaidu_crawler/woaidu_crawler/commands/init_sharding_mongodb.py:272"},{"rule_id":"opengrep/python.lang.security.audit.dangerous-subprocess-use-audit","severity":"high","message":"Detected subprocess function 'Popen' without a static string. If this data can be controlled by a malicious actor, it may be an instance of command injection. Audit the use of this call to ensure it is not controllable by an external resource. You may consider using 'shlex.escape()'. [confidence low]","path":"woaidu_crawler/woaidu_crawler/commands/init_sharding_mongodb.py","line":284,"fingerprint":"python.lang.security.audit.dangerous-subprocess-use-audit:woaidu_crawler/woaidu_crawler/commands/init_sharding_mongodb.py:284"},{"rule_id":"opengrep/python.lang.security.audit.dangerous-subprocess-use-tainted-env-args","severity":"high","message":"Detected subprocess function 'Popen' with user controlled data. A malicious actor could leverage this to perform command injection. You may consider using 'shlex.quote()'. [confidence medium]","path":"woaidu_crawler/woaidu_crawler/commands/init_sharding_mongodb.py","line":284,"fingerprint":"python.lang.security.audit.dangerous-subprocess-use-tainted-env-args:woaidu_crawler/woaidu_crawler/commands/init_sharding_mongodb.py:284"},{"rule_id":"opengrep/python.lang.security.audit.dangerous-subprocess-use-audit","severity":"high","message":"Detected subprocess function 'Popen' without a static string. If this data can be controlled by a malicious actor, it may be an instance of command injection. Audit the use of this call to ensure it is not controllable by an external resource. You may consider using 'shlex.escape()'. [confidence low]","path":"woaidu_crawler/woaidu_crawler/commands/init_sharding_mongodb.py","line":303,"fingerprint":"python.lang.security.audit.dangerous-subprocess-use-audit:woaidu_crawler/woaidu_crawler/commands/init_sharding_mongodb.py:303"},{"rule_id":"opengrep/python.lang.security.audit.dangerous-subprocess-use-tainted-env-args","severity":"high","message":"Detected subprocess function 'Popen' with user controlled data. A malicious actor could leverage this to perform command injection. You may consider using 'shlex.quote()'. [confidence medium]","path":"woaidu_crawler/woaidu_crawler/commands/init_sharding_mongodb.py","line":303,"fingerprint":"python.lang.security.audit.dangerous-subprocess-use-tainted-env-args:woaidu_crawler/woaidu_crawler/commands/init_sharding_mongodb.py:303"},{"rule_id":"opengrep/python.lang.security.insecure-hash-algorithm-sha1","severity":"medium","message":"Detected SHA1 hash algorithm which is considered insecure. SHA1 is not collision resistant and is therefore not suitable as a cryptographic signature. Use SHA256 or SHA3 instead. [confidence medium]","path":"woaidu_crawler/woaidu_crawler/pipelines/file.py","line":242,"fingerprint":"python.lang.security.insecure-hash-algorithm-sha1:woaidu_crawler/woaidu_crawler/pipelines/file.py:242"},{"rule_id":"opengrep/python.lang.security.insecure-hash-algorithm-sha1","severity":"medium","message":"Detected SHA1 hash algorithm which is considered insecure. SHA1 is not collision resistant and is therefore not suitable as a cryptographic signature. Use SHA256 or SHA3 instead. [confidence medium]","path":"woaidu_crawler/woaidu_crawler/pipelines/mongodb_book_file.py","line":257,"fingerprint":"python.lang.security.insecure-hash-algorithm-sha1:woaidu_crawler/woaidu_crawler/pipelines/mongodb_book_file.py:257"},{"rule_id":"opengrep/python.lang.security.deserialization.avoid-cPickle","severity":"medium","message":"Avoid using `cPickle`, which is known to lead to code execution vulnerabilities. When unpickling, the serialized data could be manipulated to run arbitrary code. Instead, consider serializing the relevant data as JSON or a similar text-based serialization format. [confidence low]","path":"woaidu_crawler/woaidu_crawler/scrapy_redis/queue.py","line":29,"fingerprint":"python.lang.security.deserialization.avoid-cPickle:woaidu_crawler/woaidu_crawler/scrapy_redis/queue.py:29"},{"rule_id":"opengrep/python.lang.security.deserialization.avoid-pickle","severity":"medium","message":"Avoid using `pickle`, which is known to lead to code execution vulnerabilities. When unpickling, the serialized data could be manipulated to run arbitrary code. Instead, consider serializing the relevant data as JSON or a similar text-based serialization format. [confidence low]","path":"woaidu_crawler/woaidu_crawler/scrapy_redis/queue.py","line":29,"fingerprint":"python.lang.security.deserialization.avoid-pickle:woaidu_crawler/woaidu_crawler/scrapy_redis/queue.py:29"},{"rule_id":"opengrep/python.lang.security.deserialization.avoid-cPickle","severity":"medium","message":"Avoid using `cPickle`, which is known to lead to code execution vulnerabilities. When unpickling, the serialized data could be manipulated to run arbitrary code. Instead, consider serializing the relevant data as JSON or a similar text-based serialization format. [confidence low]","path":"woaidu_crawler/woaidu_crawler/scrapy_redis/queue.py","line":33,"fingerprint":"python.lang.security.deserialization.avoid-cPickle:woaidu_crawler/woaidu_crawler/scrapy_redis/queue.py:33"},{"rule_id":"opengrep/python.lang.security.deserialization.avoid-pickle","severity":"medium","message":"Avoid using `pickle`, which is known to lead to code execution vulnerabilities. When unpickling, the serialized data could be manipulated to run arbitrary code. Instead, consider serializing the relevant data as JSON or a similar text-based serialization format. [confidence low]","path":"woaidu_crawler/woaidu_crawler/scrapy_redis/queue.py","line":33,"fingerprint":"python.lang.security.deserialization.avoid-pickle:woaidu_crawler/woaidu_crawler/scrapy_redis/queue.py:33"}],"evidence":[],"data":{"rulesets":["python"]},"error":null},{"analyzer":"sec-deps","pillar":"security","status":"skipped","tool_version":null,"metrics":{},"findings":[],"evidence":[],"data":{},"error":null},{"analyzer":"sec-actions","pillar":"security","status":"skipped","tool_version":null,"metrics":{},"findings":[],"evidence":[],"data":{},"error":null},{"analyzer":"sec-posture","pillar":"security","status":"ok","tool_version":null,"metrics":{"check:security_policy":0.0,"check:dependency_updates":0.0,"check:signed_commits":0.0,"check:lockfiles":1.0,"signed_commit_share":0.0,"unpinned_manifests":0.0},"findings":[],"evidence":[],"data":{},"error":null},{"analyzer":"sec-distribution","pillar":"security","status":"ok","tool_version":null,"metrics":{"signals":0.0,"binary_assets":0.0,"suspicious":0.0},"findings":[],"evidence":[],"data":{"signals":[],"suspicious":false,"unusual_traction":false,"metadata":true},"error":null},{"analyzer":"qual-complexity","pillar":"quality","status":"ok","tool_version":"lizard 1.25.1","metrics":{"functions":150.0,"function_nloc":957.0,"mean_ccn":2.21,"p90_ccn":5.0,"complex_functions":0.0,"long_functions":0.0,"many_param_functions":0.0,"risky_nloc_share":0.0},"findings":[],"evidence":[],"data":{},"error":null},{"analyzer":"qual-duplication","pillar":"quality","status":"ok","tool_version":"repotrust-dup-w6","metrics":{"compared_lines":1850.0,"duplicated_lines":360.0,"duplicated_share":0.1946,"files":32.0},"findings":[],"evidence":[{"kind":"duplicate-code","description":"6+ duplicated lines: woaidu_crawler/woaidu_crawler/pipelines/bookfile.py:1 and woaidu_crawler/woaidu_crawler/pipelines/mongodb_book_file.py:1","ref":"woaidu_crawler/woaidu_crawler/pipelines/bookfile.py","url":"https://github.com/gnemoug/distribute_crawler/blob/fb74ad853126b3a6358f987d763d492d8f3eec12/woaidu_crawler/woaidu_crawler/pipelines/bookfile.py#L1","weight":0.3},{"kind":"duplicate-code","description":"6+ duplicated lines: woaidu_crawler/woaidu_crawler/pipelines/bookfile.py:121 and woaidu_crawler/woaidu_crawler/pipelines/cover_image.py:23","ref":"woaidu_crawler/woaidu_crawler/pipelines/bookfile.py","url":"https://github.com/gnemoug/distribute_crawler/blob/fb74ad853126b3a6358f987d763d492d8f3eec12/woaidu_crawler/woaidu_crawler/pipelines/bookfile.py#L121","weight":0.3},{"kind":"duplicate-code","description":"6+ duplicated lines: woaidu_crawler/woaidu_crawler/pipelines/file.py:140 and woaidu_crawler/woaidu_crawler/pipelines/mongodb_book_file.py:175","ref":"woaidu_crawler/woaidu_crawler/pipelines/file.py","url":"https://github.com/gnemoug/distribute_crawler/blob/fb74ad853126b3a6358f987d763d492d8f3eec12/woaidu_crawler/woaidu_crawler/pipelines/file.py#L140","weight":0.3},{"kind":"duplicate-code","description":"6+ duplicated lines: woaidu_crawler/woaidu_crawler/pipelines/mongodb.py:21 and woaidu_crawler/woaidu_crawler/pipelines/mongodb.py:78","ref":"woaidu_crawler/woaidu_crawler/pipelines/mongodb.py","url":"https://github.com/gnemoug/distribute_crawler/blob/fb74ad853126b3a6358f987d763d492d8f3eec12/woaidu_crawler/woaidu_crawler/pipelines/mongodb.py#L21","weight":0.3},{"kind":"duplicate-code","description":"6+ duplicated lines: woaidu_crawler/woaidu_crawler/scrapy_redis/queue.py:57 and woaidu_crawler/woaidu_crawler/scrapy_redis/queue.py:99","ref":"woaidu_crawler/woaidu_crawler/scrapy_redis/queue.py","url":"https://github.com/gnemoug/distribute_crawler/blob/fb74ad853126b3a6358f987d763d492d8f3eec12/woaidu_crawler/woaidu_crawler/scrapy_redis/queue.py#L57","weight":0.3},{"kind":"duplicate-code","description":"6+ duplicated lines: woaidu_crawler/woaidu_crawler/statscol/graphite.py:39 and woaidu_crawler/woaidu_crawler/statscol/graphite.py:169","ref":"woaidu_crawler/woaidu_crawler/statscol/graphite.py","url":"https://github.com/gnemoug/distribute_crawler/blob/fb74ad853126b3a6358f987d763d492d8f3eec12/woaidu_crawler/woaidu_crawler/statscol/graphite.py#L39","weight":0.3}],"data":{},"error":null},{"analyzer":"qual-lint","pillar":"quality","status":"ok","tool_version":"opengrep 1.33.1; rules f1d2b562b414","metrics":{"findings":8.0,"weighted_findings":18.0,"scanned_lines":2604.0,"category:best-practice":7.0,"category:maintainability":1.0},"findings":[{"rule_id":"opengrep/python.lang.best-practice.open-never-closed","severity":"high","message":"file object opened without corresponding close","path":"woaidu_crawler/woaidu_crawler/commands/init_sharding_mongodb.py","line":196,"fingerprint":"python.lang.best-practice.open-never-closed:woaidu_crawler/woaidu_crawler/commands/init_sharding_mongodb.py:196"},{"rule_id":"opengrep/python.lang.best-practice.arbitrary-sleep","severity":"high","message":"time.sleep() call; did you mean to leave this in?","path":"woaidu_crawler/woaidu_crawler/commands/init_sharding_mongodb.py","line":229,"fingerprint":"python.lang.best-practice.arbitrary-sleep:woaidu_crawler/woaidu_crawler/commands/init_sharding_mongodb.py:229"},{"rule_id":"opengrep/python.lang.best-practice.arbitrary-sleep","severity":"high","message":"time.sleep() call; did you mean to leave this in?","path":"woaidu_crawler/woaidu_crawler/commands/init_sharding_mongodb.py","line":240,"fingerprint":"python.lang.best-practice.arbitrary-sleep:woaidu_crawler/woaidu_crawler/commands/init_sharding_mongodb.py:240"},{"rule_id":"opengrep/python.lang.best-practice.arbitrary-sleep","severity":"high","message":"time.sleep() call; did you mean to leave this in?","path":"woaidu_crawler/woaidu_crawler/commands/init_sharding_mongodb.py","line":246,"fingerprint":"python.lang.best-practice.arbitrary-sleep:woaidu_crawler/woaidu_crawler/commands/init_sharding_mongodb.py:246"},{"rule_id":"opengrep/python.lang.best-practice.arbitrary-sleep","severity":"high","message":"time.sleep() call; did you mean to leave this in?","path":"woaidu_crawler/woaidu_crawler/commands/init_sharding_mongodb.py","line":320,"fingerprint":"python.lang.best-practice.arbitrary-sleep:woaidu_crawler/woaidu_crawler/commands/init_sharding_mongodb.py:320"},{"rule_id":"opengrep/python.lang.best-practice.unspecified-open-encoding","severity":"medium","message":"Missing 'encoding' parameter. 'open()' uses device locale encodings by default, corrupting files with special characters. Specify the encoding to ensure cross-platform support when opening files in text mode (e.g. encoding=\"utf-8\").","path":"woaidu_crawler/woaidu_crawler/commands/init_sharding_mongodb.py","line":196,"fingerprint":"python.lang.best-practice.unspecified-open-encoding:woaidu_crawler/woaidu_crawler/commands/init_sharding_mongodb.py:196"},{"rule_id":"opengrep/python.lang.best-practice.unspecified-open-encoding","severity":"medium","message":"Missing 'encoding' parameter. 'open()' uses device locale encodings by default, corrupting files with special characters. Specify the encoding to ensure cross-platform support when opening files in text mode (e.g. encoding=\"utf-8\").","path":"woaidu_crawler/woaidu_crawler/pipelines/file.py","line":46,"fingerprint":"python.lang.best-practice.unspecified-open-encoding:woaidu_crawler/woaidu_crawler/pipelines/file.py:46"},{"rule_id":"opengrep/python.lang.maintainability.return-not-in-function","severity":"medium","message":"`return` only makes sense inside a function","path":"woaidu_crawler/woaidu_crawler/utils/select_result.py","line":63,"fingerprint":"python.lang.maintainability.return-not-in-function:woaidu_crawler/woaidu_crawler/utils/select_result.py:63"}],"evidence":[],"data":{},"error":null},{"analyzer":"qual-testing","pillar":"quality","status":"ok","tool_version":null,"metrics":{"check:has_tests":1.0,"check:test_ratio":0.023,"check:ci_present":0.0,"check:ci_runs_tests":0.0,"check:ci_runs_lint":0.0,"check:tests_cochange":1.0,"test_lines":30.0,"source_lines":2604.0,"test_ratio":0.0115,"ci_files":0.0,"cochange_share":0.25,"cochange_commits":4.0},"findings":[],"evidence":[],"data":{"test_commands":[],"lint_commands":[]},"error":null},{"analyzer":"qual-docs","pillar":"quality","status":"ok","tool_version":null,"metrics":{"check:readme":1.0,"check:license":0.0,"check:contributing":0.0,"check:changelog":0.0,"readme_lines":119.0},"findings":[],"evidence":[{"kind":"docs","description":"README.md","ref":"README.md","url":null,"weight":0.0}],"data":{},"error":null},{"analyzer":"qual-type-safety","pillar":"quality","status":"ok","tool_version":null,"metrics":{"check:strict_config":0.0,"check:low_escape_density":1.0,"escapes":1.0,"escapes_per_kloc":0.384},"findings":[],"evidence":[],"data":{},"error":null}],"flags":[]}