From 69e6ba2392bb37b63cf7ac5126cb2e020453a5f3 Mon Sep 17 00:00:00 2001 From: wyang Date: Thu, 18 Jan 2024 17:14:36 +0800 Subject: [PATCH] override DefaultSerializer, debug unserialized --- .github/workflows/codeql.yaml | 44 ------------ .github/workflows/dependencies.yml | 109 ----------------------------- .github/workflows/docker.yml | 55 --------------- .github/workflows/lint.yml | 37 ---------- .github/workflows/workflow.yml | 107 ---------------------------- rq/job.py | 3 +- rq/serializers.py | 9 ++- 7 files changed, 9 insertions(+), 355 deletions(-) delete mode 100644 .github/workflows/codeql.yaml delete mode 100644 .github/workflows/dependencies.yml delete mode 100644 .github/workflows/docker.yml delete mode 100644 .github/workflows/lint.yml delete mode 100644 .github/workflows/workflow.yml diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml deleted file mode 100644 index 07b94a0..0000000 --- a/.github/workflows/codeql.yaml +++ /dev/null @@ -1,44 +0,0 @@ -name: "Code Scanning - Action" - -on: - pull_request: - push: - -jobs: - CodeQL-Build: - # CodeQL runs on ubuntu-latest, windows-latest, and macos-latest - runs-on: ubuntu-20.04 - - permissions: - # required for all workflows - security-events: write - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - # Override language selection by uncommenting this and choosing your languages - # with: - # languages: go, javascript, csharp, python, cpp, java - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below). - - name: Autobuild - uses: github/codeql-action/autobuild@v2 - - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - - # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following - # three lines and modify them (or add more) to build your code if your - # project uses a compiled language - - #- run: | - # make bootstrap - # make release - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml deleted file mode 100644 index b1807f7..0000000 --- a/.github/workflows/dependencies.yml +++ /dev/null @@ -1,109 +0,0 @@ -name: Dependencies - -on: - schedule: - # View https://docs.github.com/en/actions/reference/events-that-trigger-workflows#schedule - - cron: '0 12 * * *' - workflow_dispatch: - -jobs: - build: - if: github.repository == 'rq/rq' - name: Python${{ matrix.python-version }}/Redis${{ matrix.redis-version }}/redis-py${{ matrix.redis-py-version }} - runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: ["3.6", "3.7", "3.8.3", "3.9", "3.10", "3.11"] - redis-version: [3, 4, 5, 6, 7] - redis-py-version: [3.5.0] - - steps: - - uses: actions/checkout@v3 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4.6.1 - with: - python-version: ${{ matrix.python-version }} - - - name: Start Redis - uses: supercharge/redis-github-action@1.5.0 - with: - redis-version: ${{ matrix.redis-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install redis==${{ matrix.redis-py-version }} - pip install -r requirements.txt -r dev-requirements.txt - pip install -e . - - - name: Test with pytest - run: | - RUN_SLOW_TESTS_TOO=1 pytest --durations=5 - - dependency-build: - name: Check development branches of dependencies - runs-on: ubuntu-latest - needs: build - if: success() - - strategy: - matrix: - python-version: ["3.6", "3.7", "3.8.3", "3.9", "3.10", "3.11"] - redis-version: [3, 4, 5, 6, 7] - - steps: - - uses: actions/checkout@v3 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4.6.1 - with: - python-version: ${{ matrix.python-version }} - - - name: Start Redis - uses: supercharge/redis-github-action@1.5.0 - with: - redis-version: ${{ matrix.redis-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install git+https://github.com/redis/redis-py - pip install git+https://github.com/pallets/click - pip install -r dev-requirements.txt - pip install -e . - - - name: Test with pytest - run: RUN_SLOW_TESTS_TOO=1 pytest --durations=5 > log.txt 2>&1 - - - uses: actions/upload-artifact@v3 - with: - name: dependencies-error - path: log.txt - if: failure() - - issue: - name: Create failure issue - runs-on: ubuntu-latest - - if: failure() - needs: dependency-build - - steps: - - uses: actions/download-artifact@v3 - with: - name: dependencies-error - path: . - - - name: Create failure issue - run: | - if [[ "$(curl --url https://api.github.com/repos/${{ github.repository }}/issues?creator=github-actions --request GET)" != *"\""* ]] - then curl --request POST \ - --url https://api.github.com/repos/${{ github.repository }}/issues \ - --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \ - --header 'content-type: application/json' \ - --data "{ - \"title\": \"RQ maybe may not work with dependencies in the future\", - \"body\": \"This issue was automatically created by the GitHub Action workflow **${{ github.workflow }}**. \n\n View log: \n\n \`\`\` \n $(cat log.txt | while read line; do echo -n "$line\n"; done | sed -r 's/"/\\"/g') \n \`\`\`\" - }" - fi diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml deleted file mode 100644 index 7a9dda6..0000000 --- a/.github/workflows/docker.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: Docker - -on: - push: - branches: [ master ] - tags: [ '*' ] - workflow_dispatch: - - -permissions: - contents: read write # to fetch code (actions/checkout) - packages: write - -jobs: - push: - if: github.repository == 'rq/rq' - runs-on: ubuntu-20.04 - - steps: - - uses: actions/checkout@v3 - - - name: Push - run: | - # Parse Version - # "master" -> "master" - # "v1.2.3" -> "1.2.3", "1.2", "1", "latest" - - VERSIONS=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') - - [[ "${{ github.ref }}" == "refs/tags/"* ]] && { - VERSIONS=$(echo $VERSIONS | sed -e 's/^v//') - i="$VERSIONS" - while [[ "$i" == *"."* ]] - do i="$(echo "$i" | sed 's/\(.*\)\..*/\1/g')" - VERSIONS="$VERSIONS $i" - done - VERSIONS="$VERSIONS latest" - } - - echo Building with tags: $VERSIONS - - # Login to registries - echo "${{ secrets.GITHUB_TOKEN }}" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin - echo "${{ secrets.DOCKER_TOKEN }}" | docker login -u selwin --password-stdin - - # Build image - docker build . --tag worker - - # Tag and Push - for VERSION in $VERSIONS - do docker tag worker redisqueue/worker:$VERSION - docker push redisqueue/worker:$VERSION - docker tag worker docker.pkg.github.com/rq/rq/worker:$VERSION - docker push docker.pkg.github.com/rq/rq/worker:$VERSION - done diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index f53343b..0000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: Lint rq - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -permissions: - contents: read - -jobs: - lint: - name: Lint - runs-on: ubuntu-20.04 - - steps: - - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4.6.1 - with: - python-version: 3.8 - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install black ruff - - - name: Lint with black - run: | - black --check --skip-string-normalization --line-length 120 rq tests - - - name: Lint with ruff - run: | - # stop the build if there are Python syntax errors. - ruff check --show-source rq tests diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml deleted file mode 100644 index 1de525f..0000000 --- a/.github/workflows/workflow.yml +++ /dev/null @@ -1,107 +0,0 @@ -name: Test - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -permissions: - contents: read # to fetch code (actions/checkout) - -jobs: - ssl-test: - name: Run SSL tests - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v3 - - name: Build the Docker test image for tox - uses: docker/build-push-action@v4 - with: - file: tests/Dockerfile - tags: rqtest-image:latest - push: false - - name: Launch tox SSL env only (will only run SSL specific tests) - uses: addnab/docker-run-action@v3 - with: - image: rqtest-image:latest - run: stunnel & redis-server & RUN_SSL_TESTS=1 tox run -e ssl - - test: - name: Python${{ matrix.python-version }}/Redis${{ matrix.redis-version }}/redis-py${{ matrix.redis-py-version }} - runs-on: ubuntu-20.04 - timeout-minutes: 10 - strategy: - matrix: - python-version: ["3.7", "3.8.3", "3.9", "3.10", "3.11"] - redis-version: [3, 4, 5, 6, 7] - redis-py-version: [3.5.0] - - steps: - - uses: actions/checkout@v3 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4.6.1 - with: - python-version: ${{ matrix.python-version }} - - - name: Start Redis - uses: supercharge/redis-github-action@1.5.0 - with: - redis-version: ${{ matrix.redis-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install redis==${{ matrix.redis-py-version }} - pip install -r requirements.txt -r dev-requirements.txt - pip install -e . - - - name: Test with pytest - run: | - RUN_SLOW_TESTS_TOO=1 pytest --cov=rq --cov-config=.coveragerc --cov-report=xml --durations=5 - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - file: ./coverage.xml - fail_ci_if_error: false - test-python-36: - name: Python${{ matrix.python-version }}/Redis${{ matrix.redis-version }}/redis-py${{ matrix.redis-py-version }} - runs-on: ubuntu-20.04 - timeout-minutes: 10 - strategy: - matrix: - python-version: ["3.6"] - redis-version: [3, 4, 5, 6, 7] - redis-py-version: [3.5.0] - - steps: - - uses: actions/checkout@v3 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4.6.1 - with: - python-version: ${{ matrix.python-version }} - - - name: Start Redis - uses: supercharge/redis-github-action@1.5.0 - with: - redis-version: ${{ matrix.redis-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install redis==${{ matrix.redis-py-version }} - pip install -r requirements.txt -r dev-requirements-36.txt - pip install -e . - - - name: Test with pytest - run: | - RUN_SLOW_TESTS_TOO=1 pytest --cov=rq --cov-config=.coveragerc --cov-report=xml --durations=5 - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - file: ./coverage.xml - fail_ci_if_error: false diff --git a/rq/job.py b/rq/job.py index f2707b5..e1c77ec 100644 --- a/rq/job.py +++ b/rq/job.py @@ -965,7 +965,8 @@ class Job: try: self.meta = self.serializer.loads(obj.get('meta')) if obj.get('meta') else {} except Exception: # depends on the serializer - self.meta = {'unserialized': obj.get('meta', {})} + import traceback + self.meta = {'unserialized': obj.get('meta', {}), 'stack': traceback.format_exc()} self.retries_left = int(obj.get('retries_left')) if obj.get('retries_left') else None if obj.get('retry_intervals'): diff --git a/rq/serializers.py b/rq/serializers.py index 94eddbf..395d184 100644 --- a/rq/serializers.py +++ b/rq/serializers.py @@ -7,8 +7,13 @@ from .utils import import_attribute class DefaultSerializer: - dumps = partial(pickle.dumps, protocol=pickle.HIGHEST_PROTOCOL) - loads = pickle.loads + @staticmethod + def dumps(*args, **kwargs): + return json.dumps(*args, **kwargs).encode('utf-8') + + @staticmethod + def loads(s, *args, **kwargs): + return json.loads(s.decode('utf-8'), *args, **kwargs) class JSONSerializer: