Merge pull request #2 from doximity/jcw/maintained_history_release

Initial release including curated git history from the original repository

.circleci/config.yml

@@ -0,0 +1,125 @@
+version: 2.1
+
+orbs:
+  gem: doximity/gem-publisher@0
+
+executors:
+  ruby-latest:
+    resource_class: small
+    docker:
+      - image: circleci/ruby:2.6
+        environment:
+          BUNDLE_VERSION: "~> 2.1"
+
+# yaml anchor filters
+master_only: &master_only
+  filters:
+    branches:
+      only: master
+    tags:
+      ignore: /.*/
+pr_only: &pr_only
+  filters:
+    branches:
+      ignore: master
+    tags:
+      ignore: /.*/
+version_tags_only: &version_tags_only
+  filters:
+    branches:
+      ignore: /.*/
+    tags:
+      only: /^v.*/
+
+jobs:
+  build:
+    executor: ruby-latest
+    steps:
+      - checkout
+      - run:
+          name: Install Bundler specific version
+          command: |
+            gem install bundler --version "${BUNDLE_VERSION}" --force
+      - restore_cache:
+          keys:
+            - v1-bundle-{{ checksum "Gemfile.lock" }}-
+      - run:
+          name: Install Ruby Dependencies
+          command: bundle check --path=vendor/bundle || bundle install --local --frozen --path=vendor/bundle --jobs=4 --retry=3
+      - save_cache:
+          key: v1-bundle-{{ checksum "Gemfile.lock" }}-
+          paths:
+            - vendor/bundle
+      - run:
+          name: Run Tests
+          command: bundle exec rake ci:specs
+      - store_test_results:
+          name: Store test results
+          path: tmp/test-results
+      - run:
+          name: Build documentation
+          command: bundle exec rake ci:doc
+      - store_artifacts:
+          name: Saves documentation
+          path: doc
+      - persist_to_workspace:
+          root: .
+          paths:
+            - vendor/bundle
+
+workflows:
+  version: 2
+
+  trunk:
+    jobs:
+      - build:
+          <<: *master_only
+      - gem/build:
+          <<: *master_only
+          executor: ruby-latest
+          name: gem-build
+          requires:
+            - build
+
+  pull-requests:
+    jobs:
+      - build:
+          <<: *pr_only
+      - gem/build:
+          <<: *pr_only
+          executor: ruby-latest
+          name: gem-build
+          requires:
+            - build
+      - pre-release-approval:
+          <<: *pr_only
+          type: approval
+          requires:
+            - gem-build
+      - gem/publish:
+          <<: *pr_only
+          name: gem-publish
+          to_rubygems: true
+          pre_release: true
+          requires:
+            - pre-release-approval
+          context: artifact_publishing
+
+  final-release:
+    jobs:
+      - build:
+          <<: *version_tags_only
+      - gem/build:
+          <<: *version_tags_only
+          executor: ruby-latest
+          name: gem-build
+          requires:
+            - build
+      - gem/publish:
+          <<: *version_tags_only
+          name: gem-publish
+          to_rubygems: true
+          pre_release: false
+          requires:
+            - gem-build
+          context: artifact_publishing

.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

CONTRIBUTING.md

@@ -0,0 +1,31 @@
+# Contributing
+
+We welcome contributions to this repository. Feel free to submit issues for bugs you encounter and pull requests for code and documentation contributions.
+
+In order to prevent licensing issues, we require all contributors to sign an individual contributor license agreement, which is reproduced below:
+
+## Individual Contributor License Agreement
+
+In order to clarify the intellectual property license granted with Contributions from any person or entity, Doximity Inc. ("Doximity") must have a Contributor License Agreement ("CLA") on file that has been signed by each Contributor, indicating agreement to the license terms below. This license is for your protection as a Contributor as well as the protection of Doximity; it does not change your rights to use your own Contributions for any other purpose.
+
+You accept and agree to the following terms and conditions for Your present and future Contributions submitted to Doximity. Except for the license granted herein to Doximity and recipients of software distributed by Doximity, You reserve all right, title, and interest in and to Your Contributions.
+
+### Definitions
+
+"You" (or "Your") shall mean the copyright owner or legal entity authorized by the copyright owner that is making this Agreement with Doximity. For legal entities, the entity making a Contribution and all other entities that control, are controlled by, or are under common control with that entity are considered to be a single Contributor. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
+
+1. "Contribution" shall mean any original work of authorship, including any modifications or additions to an existing work, that is intentionally submitted by You to Doximity for inclusion in, or documentation of, any of the products owned or managed by Doximity (the "Work"). For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to Doximity or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, Doximity for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by You as "Not a Contribution."
+
+2. Grant of Copyright License. Subject to the terms and conditions of this Agreement, You hereby grant to Doximity and to recipients of software distributed by Doximity a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, sublicense, and distribute Your Contributions and such derivative works.
+
+3. Grant of Patent License. Subject to the terms and conditions of this Agreement, You hereby grant to Doximity and to recipients of software distributed by Doximity a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by You that are necessarily infringed by Your Contribution(s) alone or by combination of Your Contribution(s) with the Work to which such Contribution(s) was submitted. If any entity institutes patent litigation against You or any other entity (including a cross-claim or counterclaim in a lawsuit) alleging that your Contribution, or the Work to which you have contributed, constitutes direct or contributory patent infringement, then any patent licenses granted to that entity under this Agreement for that Contribution or Work shall terminate as of the date such litigation is filed.
+
+4. You represent that you are legally entitled to grant the above license. If your employer(s) has rights to intellectual property that you create that includes your Contributions, you represent that you have received permission to make Contributions on behalf of that employer, that your employer has waived such rights for your Contributions to Doximity, or that your employer has executed a separate Corporate CLA with Doximity.
+
+5. You represent that each of Your Contributions is Your original creation (see section 7 for submissions on behalf of others). You represent that Your Contribution submissions include complete details of any third-party license or other restriction (including, but not limited to, related patents and trademarks) of which you are personally aware and which are associated with any part of Your Contributions.
+
+6. You are not expected to provide support for Your Contributions, except to the extent You desire to provide support. You may provide support for free, for a fee, or not at all. Unless required by applicable law or agreed to in writing, You provide Your Contributions on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON- INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE.
+
+7. Should You wish to submit work that is not Your original creation, You may submit it to Doximity separately from any Contribution, identifying the complete details of its source and of any license or other restriction (including, but not limited to, related patents, trademarks, and license agreements) of which you are personally aware, and conspicuously marking the work as "Submitted on behalf of a third-party: [[]named here]".
+
+8. You agree to notify Doximity of any facts or circumstances of which you become aware that would make these representations inaccurate in any respect.

CONTRIBUTORS.md

@@ -0,0 +1,19 @@
+## List of All Known Code Contributors to Simplekiq
+
+### Jack Noble
+* Collaborated on initial concept
+* Wrote the majority of the code as of initial release
+
+### John Wilkinson
+* Collaborated on initial concept
+* Conducted the gem extraction and release
+
+### Brian Dillard
+* Added additional comment documentation
+* Added support for `on_complete` batch callback support in `Simplekiq::BatchingJob`
+
+### Austin Madden
+* Fixed bug with batch statuses in callbacks for empty batches
+
+### Tiffany Troha
+* Added support for specifying `sidekiq_options` for the child job in `Simplekiq::BatchingJob`

Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in simplekiq.gemspec
+gemspec

LICENSE.txt

@@ -0,0 +1,13 @@
+Copyright 2022 Doximity, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

README.md

@@ -1,2 +1,125 @@
-# simplekiq
-Sidekiq-based workflow orchestration library
+# Simplekiq
+
+Any time that you find yourself needing to string together a long chain of jobs, particularly when there are multiple stages of Sidekiq-pro batches and callbacks involved, come home instead to the simple flavor of orchestrated job flow with Simplekiq.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'simplekiq'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install simplekiq
+
+## Usage
+
+There are currently two primary components of the system which were designed to work in harmony:
+
+* [Simplekiq::OrchestrationJob](./lib/simplekiq/orchestration_job.rb) - A mixin for a Sidekiq jobs to be able to orchestrate a flow of jobs in one place. It makes long complicated flows between jobs easier to understand, iterate on, and test. It eliminates the need to hop between dozens of files to determine when, where, and why a particular job gets called.
+* [Simplekiq::BatchingJob](./lib/simplekiq/batching_job.rb) - A mixin designed to make breaking a large job into a batched process dead simple and contained within a single class while still being trivially composable in orchestrations.
+
+## Tool Drilldown
+
+### Simplekiq::OrchestrationJob
+
+Mixing in the [Simplekiq::Orchestration](./lib/simplekiq/orchestration_job.rb) module lets you define a human-readable workflow of jobs in a single file with almost* no special requirements or restrictions on how the child jobs are designed. In most cases, Sidekiq jobs not designed for use in orchestrations should be compatible for use in orchestrations. A job implementing `OrchestrationJob` might look like:
+
+```ruby
+class SomeOrchestrationJob < BaseJob
+  include Sidekiq::Worker
+  include Simplekiq::OrchestrationJob
+
+  def perform_orchestration(some_id)
+    @some_model = SomeModel.find(some_id) # 1.
+
+    run SomeInitialSetupJob, some_model.id # 2.
+
+    in_parallel do
+      some_related_models.each do |related_model|
+        run SomeParallelizableJob, related_model.id # 3.
+      end
+    end
+
+    run SomeFinalizationJob, some_model.id # 4.
+  end
+
+  private
+
+  attr_reader :some_model
+
+  def some_related_models
+    @some_related_models ||= some_model.some_relation
+  end
+end
+```
+
+Let's use the above example to describe some specifics of how the flow works.
+
+1. `SomeOrchestrationJob` pulls up some instance of parent model `SomeModel`.
+2. It does some initial work in `SomeInitialSetupJob`, which blocks the rest of the workflow until it completes successfully.
+3. Then it will run a `SomeParallelizableJob` for each of some number of associated models `some_related_models`. These jobs will all run parallel to each other independently.
+4. Finally, after all of the parallel jobs from #3 complete successfully, `SomeFinalizationJob` will run and then after it finishes the orchestration will be complete.
+
+**Note** - it's fine to add utility methods and `attr_accessor`s to keep the code tidy and maintainable.
+
+When `SomeOrchestrationJob` itself gets called though, the first thing it does it turn these directives into a big serialized structure indicating which job will be called under what conditions (eg, serial or in parallel) and with what arguments, and then keeps passing that between the simplekiq-internal jobs that actually conduct the flow.
+
+This means when you want to deploy a change to this flow all previous in-flight workflows will continue undisturbed because the workflow is frozen in sidekiq job arguments and will remain frozen until the workflow completes. This is generally a boon, but note that if you remove a job from a workflow you'll need to remember to either keep the job itself (eg, the `SomeFinalizationJob` class file from our above example) in the codebase or replace it with a stub so that any in-flight workflows won't crash due to not being able to pull up the prior-specified workflow.
+
+"almost* no special requirements or restrictions on how the child jobs are designed" - The one thing you'll want to keep in mind when feeding arbitrary jobs into orchestrations is that if the job creates any new sidekiq batches then those new sidekiq batches should be added as child sidekiq batches of the parent sidekiq batch of the job. The parent sidekiq batch of the job is the sidekiq batch that drives the orchestration from step to step, so if you don't do this it will move onto the next step in the orchestration once your job finishes even if the new sidekiq batches it started didn't finish. This sounds more complicated than it is, you can see an example of code that does this in [`BatchingJob#perform`](./lib/simplekiq/batching_job.rb):
+
+```ruby
+if batch # is there a parent batch?
+  batch.jobs do # open the parent batch back up
+    create_a_new_batch_and_add_jobs_to_it_to_run # make our new batch as a child batch of the parent batch
+  end # close the parent batch again
+else # there's no parent batches, this job was run directly outside of an orchestration
+  create_a_new_batch_and_add_jobs_to_it_to_run # make our new batch without a parent batch
+end
+```
+
+### Simplekiq::BatchingJob
+
+See the [Simplekiq::BatchingJob](./lib/simplekiq/batching_job.rb) module itself for a description and example usage in the header comments. Nutshell is that you should use this if you're planning on making a batched asynchronous process as it shaves off a lot of ceremony and unexpressive structure. eg - Instead of having `BeerBottlerJob` which queues some number of `BeerBottlerBatchJob`s to handle the broken down sub-tasks you can just have `BeerBottlerJob` with a method for batching, executing individual batches, and a callback that gets run after all batches have completed successfully.
+
+## History
+
+Simplekiq was initially released for private use within Doximity applications in Oct 2020 where it continued to be iterated on towards stability and general use until Jan 2022 when it was deemed settled enough for public release.
+
+The primary driving factor that inspired this work was a series of over a dozen differently defined and structured jobs part of a single workflow of which the logical flow was extraordinarily difficult to cognitively trace. This led to exteme difficulty in debugging and following problematic instances of the workflow in production as well as needlessly high cost to refactoring and iterative adjustments.
+
+The crux of the problem was that each job was highly coupled to its position in the overall flow as well as the absence of any central mechanism to indicate what the overall flow was. After building Simplekiq and implementing it into the flow, significant changes to the flow became quick adjustments requiring only a couple lines of code to change and folks unfamiliar with the system could quickly get up to speed by reading through the orchestration job.
+
+## Versioning
+
+This project follows semantic versioning. At time of writing it is sitting at 0.0.1 until its integration with the application it was extracted from is confirmed to be stable. Once confirmed it will be started off at 1.0.0 as it has otherwise been used in a production system already for some time.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Note that this depends on `sidekiq-pro` which requires a [commercial license](https://sidekiq.org/products/pro.html) to install and use.
+
+Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+TODO: Update this section with more specific/appropriate instructions once this is a public repository.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request
+6. Sign the CLA if you haven't yet. See CONTRIBUTING.md
+
+## License
+
+The gem is licensed under an Apache 2 license. Contributors are required to sign an contributor license agreement. See LICENSE.txt and CONTRIBUTING.md for more information.

Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "simplekiq"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

lib/simplekiq.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "sidekiq"
+require "sidekiq-ent"
+
+require "simplekiq/orchestration_executor"
+require "simplekiq/orchestration"
+require "simplekiq/orchestration_job"
+require "simplekiq/batching_job"
+
+module Simplekiq
+end