feat: upgrade to SDK 2.1.8 with lazy loading and improved committer handling

- Upgrade socket-sdk-python dependency to version 2.1.8 to support lazy file loading capabilities
- Enable lazy file loading in fullscans.post() with use_lazy_loading=True and max_open_files=50 to prevent "Too many open files" errors when processing large numbers of manifest files
- Remove custom lazy_file_loader module as this functionality is now handled by the SDK
- Fix committer display format by implementing proper priority order:
  1. CLI --committers argument (highest priority)
  2. CI/CD SCM username (GITHUB_ACTOR, GITLAB_USER_LOGIN, BITBUCKET_STEP_TRIGGERER_UUID)
  3. Git username extracted from email patterns (e.g., GitHub noreply emails)
  4. Git email address
  5. Git author name (fallback)
- Add get_formatted_committer() method to Git class to properly format committer strings instead of displaying raw git.Actor objects
- Include license alerts in diff processing by removing licenseSpdxDisj filter condition
- Change ulimit warning messages from log.warning to log.debug to reduce noise
- Update create_full_scan() method signature to accept file paths directly instead of pre-processed file objects
- Remove deprecated load_files_for_sending() method as lazy loading is now handled by the SDK

This update improves performance for large repositories, provides better committer identification in CI/CD environments, and ensures license violations are properly reported.

Author: dacoburn

pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "socketsecurity"
-version = "2.1.35"
+version = "2.1.36"
 requires-python = ">= 3.10"
 license = {"file" = "LICENSE"}
 dependencies = [
@@ -16,7 +16,7 @@ dependencies = [
     'GitPython',
     'packaging',
     'python-dotenv',
-    'socket-sdk-python>=2.1.5,<3'
+    'socket-sdk-python>=2.1.8,<3'
 ]
 readme = "README.md"
 description = "Socket Security CLI for CI/CD"

requirements.txt

@@ -59,7 +59,7 @@ requests==2.32.4
     # via socketsecurity
 smmap==5.0.2
     # via gitdb
-socket-sdk-python==2.1.5
+socket-sdk-python==2.1.8
     # via socketsecurity
 typing-extensions==4.12.2
     # via socket-sdk-python

socketsecurity/__init__.py

@@ -1,2 +1,2 @@
 __author__ = 'socket.dev'
-__version__ = '2.1.35'
+__version__ = '2.1.36'

socketsecurity/core/__init__.py

@@ -30,7 +30,6 @@
 from .socket_config import SocketConfig
 from .utils import socket_globs
 from .resource_utils import check_file_count_against_ulimit
-from .lazy_file_loader import load_files_for_sending_lazy
 import importlib
 logging_std = importlib.import_module("logging")
 
@@ -338,10 +337,10 @@ def find_files(self, path: str) -> List[str]:
         ulimit_check = check_file_count_against_ulimit(file_count)
         if ulimit_check["can_check"]:
             if ulimit_check["would_exceed"]:
-                log.warning(f"Found {file_count} manifest files, which may exceed the file descriptor limit (ulimit -n = {ulimit_check['soft_limit']})")
-                log.warning(f"Available file descriptors: {ulimit_check['available_fds']} (after {ulimit_check['buffer_size']} buffer)")
-                log.warning(f"Recommendation: {ulimit_check['recommendation']}")
-                log.warning("This may cause 'Too many open files' errors during processing")
+                log.debug(f"Found {file_count} manifest files, which may exceed the file descriptor limit (ulimit -n = {ulimit_check['soft_limit']})")
+                log.debug(f"Available file descriptors: {ulimit_check['available_fds']} (after {ulimit_check['buffer_size']} buffer)")
+                log.debug(f"Recommendation: {ulimit_check['recommendation']}")
+                log.debug("This may cause 'Too many open files' errors during processing")
             else:
                 log.debug(f"File count ({file_count}) is within file descriptor limit ({ulimit_check['soft_limit']})")
         else:
@@ -441,30 +440,12 @@ def empty_head_scan_file() -> list[tuple[str, tuple[str, Union[BinaryIO, BytesIO
         empty_full_scan_file = [(empty_filename, (empty_filename, empty_file_obj))]
         return empty_full_scan_file
 
-    @staticmethod
-    def load_files_for_sending(files: List[str], workspace: str) -> List[Tuple[str, Tuple[str, BinaryIO]]]:
-        """
-        Prepares files for sending to the Socket API using lazy loading.
-        
-        This version uses lazy file loading to prevent "Too many open files" errors
-        when processing large numbers of manifest files.
-
-        Args:
-            files: List of file paths from find_files()
-            workspace: Base directory path to make paths relative to
-
-        Returns:
-            List of tuples formatted for requests multipart upload:
-            [(field_name, (filename, file_object)), ...]
-        """
-        return load_files_for_sending_lazy(files, workspace)
-
-    def create_full_scan(self, files: list[tuple[str, tuple[str, BytesIO]]], params: FullScanParams) -> FullScan:
+    def create_full_scan(self, files: List[str], params: FullScanParams) -> FullScan:
         """
         Creates a new full scan via the Socket API.
 
         Args:
-            files: List of files to scan
+            files: List of file paths to scan
             params: Parameters for the full scan
 
         Returns:
@@ -473,7 +454,7 @@ def create_full_scan(self, files: list[tuple[str, tuple[str, BytesIO]]], params:
         log.info("Creating new full scan")
         create_full_start = time.time()
 
-        res = self.sdk.fullscans.post(files, params, use_types=True)
+        res = self.sdk.fullscans.post(files, params, use_types=True, use_lazy_loading=True, max_open_files=50)
         if not res.success:
             log.error(f"Error creating full scan: {res.message}, status: {res.status}")
             raise Exception(f"Error creating full scan: {res.message}, status: {res.status}")
@@ -525,14 +506,13 @@ def create_full_scan_with_report_url(
         if save_manifest_tar_path and files:
             self.save_manifest_tar(files, save_manifest_tar_path, path)
 
-        files_for_sending = self.load_files_for_sending(files, path)
         if not files:
             return diff
 
         try:
             # Create new scan
             new_scan_start = time.time()
-            new_full_scan = self.create_full_scan(files_for_sending, params)
+            new_full_scan = self.create_full_scan(files, params)
             new_scan_end = time.time()
             log.info(f"Total time to create new full scan: {new_scan_end - new_scan_start:.2f}")
         except APIFailure as e:
@@ -779,7 +759,15 @@ def get_added_and_removed_packages(
         log.info(f"Comparing scans - Head scan ID: {head_full_scan_id}, New scan ID: {new_full_scan_id}")
         diff_start = time.time()
         try:
-            diff_report = self.sdk.fullscans.stream_diff(self.config.org_slug, head_full_scan_id, new_full_scan_id, use_types=True).data
+            diff_report = (
+                self.sdk.fullscans.stream_diff
+                           (
+                    self.config.org_slug,
+                    head_full_scan_id,
+                    new_full_scan_id,
+                    use_types=True
+                ).data
+            )
         except APIFailure as e:
             log.error(f"API Error: {e}")
             sys.exit(1)
@@ -877,7 +865,6 @@ def create_new_diff(
         if save_manifest_tar_path and files:
             self.save_manifest_tar(files, save_manifest_tar_path, path)
 
-        files_for_sending = self.load_files_for_sending(files, path)
         if not files:
             return Diff(id="NO_DIFF_RAN", diff_url="", report_url="")
 
@@ -901,7 +888,7 @@ def create_new_diff(
         # Create new scan
         try:
             new_scan_start = time.time()
-            new_full_scan = self.create_full_scan(files_for_sending, params)
+            new_full_scan = self.create_full_scan(files, params)
             new_scan_end = time.time()
             log.info(f"Total time to create new full scan: {new_scan_end - new_scan_start:.2f}")
         except APIFailure as e:
@@ -1156,11 +1143,10 @@ def add_package_alerts_to_collection(self, package: Package, alerts_collection:
                 action = self.config.security_policy[alert.type]['action']
                 setattr(issue_alert, action, True)
 
-            if issue_alert.type != 'licenseSpdxDisj':
-                if issue_alert.key not in alerts_collection:
-                    alerts_collection[issue_alert.key] = [issue_alert]
-                else:
-                    alerts_collection[issue_alert.key].append(issue_alert)
+            if issue_alert.key not in alerts_collection:
+                alerts_collection[issue_alert.key] = [issue_alert]
+            else:
+                alerts_collection[issue_alert.key].append(issue_alert)
 
         return alerts_collection
 

socketsecurity/core/git_interface.py

@@ -319,6 +319,67 @@ def commit_str(self) -> str:
         """Return commit SHA as a string"""
         return self.commit.hexsha
 
+    def get_formatted_committer(self) -> str:
+        """
+        Get the committer in the preferred order:
+        1. CLI --committers (handled in socketcli.py)
+        2. CI/CD SCM username (GitHub/GitLab/BitBucket environment variables)
+        3. Git username (extracted from email patterns like GitHub noreply)
+        4. Git email address
+        5. Git author name (fallback)
+        
+        Returns:
+            Formatted committer string
+        """
+        # Check for CI/CD environment usernames first
+        # GitHub Actions
+        github_actor = os.getenv('GITHUB_ACTOR')
+        if github_actor:
+            log.debug(f"Using GitHub actor as committer: {github_actor}")
+            return github_actor
+        
+        # GitLab CI
+        gitlab_user_login = os.getenv('GITLAB_USER_LOGIN')
+        if gitlab_user_login:
+            log.debug(f"Using GitLab user login as committer: {gitlab_user_login}")
+            return gitlab_user_login
+        
+        # Bitbucket Pipelines
+        bitbucket_step_triggerer_uuid = os.getenv('BITBUCKET_STEP_TRIGGERER_UUID')
+        if bitbucket_step_triggerer_uuid:
+            log.debug(f"Using Bitbucket step triggerer as committer: {bitbucket_step_triggerer_uuid}")
+            return bitbucket_step_triggerer_uuid
+        
+        # Fall back to commit author/committer details
+        # Priority 3: Try to extract git username from email patterns first
+        if self.author and self.author.email and self.author.email.strip():
+            email = self.author.email.strip()
+            
+            # If it's a GitHub noreply email, try to extract username
+            if email.endswith('@users.noreply.github.com'):
+                # Pattern: number+username@users.noreply.github.com
+                email_parts = email.split('@')[0]
+                if '+' in email_parts:
+                    username = email_parts.split('+')[1]
+                    log.debug(f"Extracted GitHub username from noreply email: {username}")
+                    return username
+        
+        # Priority 4: Use email if available
+        if self.author and self.author.email and self.author.email.strip():
+            email = self.author.email.strip()
+            log.debug(f"Using commit author email as committer: {email}")
+            return email
+        
+        # Priority 5: Fall back to author name as last resort
+        if self.author and self.author.name and self.author.name.strip():
+            name = self.author.name.strip()
+            log.debug(f"Using commit author name as fallback committer: {name}")
+            return name
+        
+        # Ultimate fallback
+        log.debug("Using fallback committer: unknown")
+        return "unknown"
+    
     def get_default_branch_name(self) -> str:
         """
         Get the default branch name from the remote origin.

socketsecurity/socketcli.py

@@ -125,7 +125,7 @@ def main_code():
         if not config.branch:
             config.branch = git_repo.branch
         if not config.committers:
-            config.committers = [git_repo.author]
+            config.committers = [git_repo.get_formatted_committer()]
         if not config.commit_message:
             config.commit_message = git_repo.commit_message
     except InvalidGitRepositoryError: