Skip to content

Commit

Permalink
Add auditree evidence locker large file check (#64)
Browse files Browse the repository at this point in the history
  • Loading branch information
alfinkel authored May 3, 2021
1 parent 0bb0057 commit f5e3ef6
Show file tree
Hide file tree
Showing 5 changed files with 150 additions and 2 deletions.
4 changes: 4 additions & 0 deletions CHANGES.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
# [0.14.0](https://github.com/ComplianceAsCode/auditree-arboretum/releases/tag/v0.14.0)

- [ADDED] Auditree evidence locker large file check has been added.

# [0.13.0](https://github.com/ComplianceAsCode/auditree-arboretum/releases/tag/v0.13.0)

- [ADDED] Auditree empty evidence check has been added.
Expand Down
2 changes: 1 addition & 1 deletion arboretum/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,4 @@
# limitations under the License.
"""Arboretum - Checking your compliance & security posture, continuously."""

__version__ = '0.13.0'
__version__ = '0.14.0'
39 changes: 39 additions & 0 deletions arboretum/auditree/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -476,6 +476,43 @@ can vary across evidences. But the default `is_empty` criteria is as follows:
from arboretum.auditree.checks.test_empty_evidence import EmptyEvidenceCheck
```

### Evidence Locker Large Files

* Class: [LargeFilesCheck][check-large-files]
* Purpose: Remote hosting services have file size constraints that if violated
will cause a remote push of the locker to be rejected. This check identifies
"large" sized files so they can be dealt with prior to reaching the remote
hosting service file size constraint.
* Behavior: Performs a check that flags all files in the evidence locker
(evidence and non-evidence) that exceed a large file threshold setting as
failures and flags all files within 20% of the threshold as warnings. The
large file threshold defaults to 50MB.
* Evidence depended upon:
* This check does not depend on any evidence specifically. It acts on **all**
files contained within an evidence locker.
* Configuration elements:
* `locker.large_file_threshold`
* Optional
* File size threshold in bytes as an integer
* Use if looking to override the default of 50MB. Otherwise do not include.
* NOTE: This is the same [evidence locker][] configuration setting is used by
the framework to produce large file execution log INFO messages.
* Example (optional) configuration:

```json
{
"locker": {
"large_file_threshold": 50000000
}
}
```

* Import statement:

```python
from arboretum.auditree.checks.test_locker_large_files import LargeFilesCheck
```

### Compliance Configuration

* Class: [ComplianceConfigCheck][check-compliance-config]
Expand Down Expand Up @@ -734,6 +771,7 @@ getting new commits. This check validates that.
[auditree-framework]: https://github.com/ComplianceAsCode/auditree-framework
[auditree-framework documentation]: https://complianceascode.github.io/auditree-framework/
[usage]: https://github.com/ComplianceAsCode/auditree-arboretum#usage
[evidence locker]: https://complianceascode.github.io/auditree-framework/design-principles.html#evidence-locker
[fetch-abandoned-evidence]: https://github.com/ComplianceAsCode/auditree-arboretum/blob/main/arboretum/auditree/fetchers/fetch_abandoned_evidence.py
[fetch-compliance-config]: https://github.com/ComplianceAsCode/auditree-arboretum/blob/main/arboretum/auditree/fetchers/fetch_compliance_config.py
[fetch-python-packages]: https://github.com/ComplianceAsCode/auditree-arboretum/blob/main/arboretum/auditree/fetchers/fetch_python_packages.py
Expand All @@ -743,6 +781,7 @@ getting new commits. This check validates that.
[fetch-branch-protection]: https://github.com/ComplianceAsCode/auditree-arboretum/blob/main/arboretum/auditree/fetchers/github/fetch_branch_protection.py
[check-abandoned-evidence]: https://github.com/ComplianceAsCode/auditree-arboretum/blob/main/arboretum/auditree/checks/test_abandoned_evidence.py
[check-empty-evidence]: https://github.com/ComplianceAsCode/auditree-arboretum/blob/main/arboretum/auditree/checks/test_empty_evidence.py
[check-large-files]: https://github.com/ComplianceAsCode/auditree-arboretum/blob/main/arboretum/auditree/checks/test_locker_large_files.py
[check-compliance-config]: https://github.com/ComplianceAsCode/auditree-arboretum/blob/main/arboretum/auditree/checks/test_compliance_config.py
[check-python-packages]: https://github.com/ComplianceAsCode/auditree-arboretum/blob/main/arboretum/auditree/checks/test_python_packages.py
[check-locker-integrity]: https://github.com/ComplianceAsCode/auditree-arboretum/blob/main/arboretum/auditree/checks/test_locker_repo_integrity.py
Expand Down
90 changes: 90 additions & 0 deletions arboretum/auditree/checks/test_locker_large_files.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
# -*- mode:python; coding:utf-8 -*-
# Copyright (c) 2021 IBM Corp. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Evidence locker large files check."""
from compliance.check import ComplianceCheck
from compliance.evidence import DAY, ReportEvidence
from compliance.locker import LF_DEFAULT, MB


class LargeFilesCheck(ComplianceCheck):
"""
Evidence locker large files check.
This check finds all large FILES in the evidence locker.
"""

@property
def title(self):
"""
Return the title of the checks.
:returns: the title of the checks
"""
return 'Large Evidence Locker Files'

@classmethod
def setUpClass(cls):
"""Initialize the check object with configuration settings."""
cls.config.add_evidences(
[
ReportEvidence(
'locker_large_files.md',
'auditree',
DAY,
'Evidence locker large files report.'
)
]
)
return cls

def test_large_files(self):
"""Check for large files."""
lf_size = self.config.get('locker.large_file_threshold', LF_DEFAULT)
warn_size = .8 * lf_size
for f_path, f_size in self.locker.get_large_files(warn_size).items():
if f_size > lf_size:
# Fail if file size is over the large file threshold
self.add_failures(
'Large Files:', f'`{f_path}` - {_size_to_str(f_size)}'
)
elif f_size > warn_size:
# Warn if file size is within 20% of the large file threshold
self.add_warnings(
'Large Files (almost):',
f'`{f_path}` - {_size_to_str(f_size)}'
)

def get_reports(self):
"""
Provide the check report name.
:returns: the report(s) generated for this check.
"""
return ['auditree/locker_large_files.md']

def get_notification_message(self):
"""
Large files check notifier.
:returns: notification dictionary.
"""
return {'body': None}


def _size_to_str(size):
formatted_size = f'{size/MB:.1f} MB'
if formatted_size == '0.0 MB':
formatted_size = f'{str(size)} Bytes'
return formatted_size
17 changes: 16 additions & 1 deletion devel.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,29 @@
"enabled": false
},
"locker": {
"repo_url": "https://github.com/my-org/my-evidence-locker-repo"
"repo_url": "https://github.com/my-org/my-evidence-locker-repo",
"large_file_threshold": 50000000
},
"notify": {
"slack": {}
},
"org": {
"name": "MY-ORG",
"auditree": {
"abandoned_evidence": {
"threshold": 1234567,
"exceptions": {
"raw/foo/evidence_bar.json": "This is a good reason",
"raw/foo/evidence_baz.json": "This is also a good reason"
},
"ignore_history": true
},
"empty_evidence": {
"exceptions": [
"raw/foo/evidence_bar.json",
"raw/foo/evidence_baz.json"
]
},
"repo_integrity": {
"branches": {
"https://github.com/ComplianceAsCode/auditree-arboretum": ["main"]
Expand Down

0 comments on commit f5e3ef6

Please sign in to comment.