Initial commit

app/workers/daemons/base.rb

@@ -0,0 +1,66 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+
+module Workers
+  module Daemons
+    class Base
+      class GetLockError < StandardError; end
+      class << self; attr_accessor :sleep_time end
+
+      attr_accessor :running
+      attr_reader :logger
+
+      def initialize
+        @running = true
+        @logger = Rails.logger
+      end
+
+      def stop
+        @running = false
+      end
+
+      def run
+        while running
+          begin
+            process
+          rescue ScriptError => e
+            raise e if is_db_connection_error?(e)
+
+            report_exception(e)
+          end
+          wait
+        end
+      end
+
+      def process
+        method_not_implemented
+      end
+
+      def wait
+        Kernel.sleep self.class.sleep_time
+      end
+
+      def is_db_connection_error?(exception)
+        exception.is_a?(Mysql2::Error::ConnectionError) || exception.cause.is_a?(Mysql2::Error)
+      end
+
+      def lock(klass, timeout)
+        res = ActiveRecord::Base.connection.exec_query("SELECT GET_LOCK('Peatio_#{klass}',#{timeout})")
+
+        # response from this query will look like this [{"GET_LOCK(id,10)"=>1}]
+        # returns 1 if the lock was obtained successfully, 0 if the attempt timed out
+        if res.to_a[0].values[0] == 1
+          begin
+            yield
+          rescue StandardError => e
+            report_exception(e)
+          ensure
+            ActiveRecord::Base.connection.exec_query("SELECT RELEASE_LOCK('Peatio_#{klass}')")
+          end
+        else
+          raise GetLockError, "Peatio_#{klass} is already running"
+        end
+      end
+    end
+  end
+end

app/workers/daemons/blockchain.rb

@@ -0,0 +1,114 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+
+module Workers
+  module Daemons
+    class Blockchain < Base
+      class Runner
+        attr_reader :ts, :thread
+
+        def initialize(blockchain, ts)
+          @blockchain = blockchain
+          @ts = ts
+          @thread = nil
+        end
+
+        def start
+          @thread ||= Thread.new do
+            bc_service = BlockchainService.new(@blockchain)
+
+            Rails.logger.info { "Processing #{@blockchain.name} blocks." }
+
+            loop do
+              begin
+                # Reset blockchain_service state.
+                bc_service.reset!
+
+                if @blockchain.reload.height + @blockchain.min_confirmations >= bc_service.latest_block_number
+                  Rails.logger.info { "Skip synchronization. No new blocks detected, height: #{@blockchain.height}, latest_block: #{bc_service.latest_block_number}." }
+                  Rails.logger.info { "Sleeping for 10 seconds" }
+                  sleep(10)
+                  next
+                end
+
+                from_block = @blockchain.height || 0
+
+                (from_block..bc_service.latest_block_number).each do |block_id|
+                  Rails.logger.info { "Started processing #{@blockchain.key} block number #{block_id}." }
+                  block_json = bc_service.process_block(block_id)
+                  Rails.logger.info { "Fetch #{block_json.transactions.count} transactions in block number #{block_id}." }
+                  bc_service.update_height(block_id)
+                  Rails.logger.info { "Finished processing #{@blockchain.key} block number #{block_id}." }
+                end
+              rescue StandardError => e
+                report_exception(e)
+                Rails.logger.warn { "Error: #{e}. Sleeping for 10 seconds" }
+                sleep(10)
+              end
+            end
+          end
+        end
+
+        def stop
+          @thread&.kill
+        end
+      end
+
+      def run
+        @runner_pool = ::Blockchain.active.each_with_object({}) do |b, pool|
+          max_ts = [b.currencies.maximum(:updated_at), b.updated_at].compact.max.to_i
+
+          logger.warn { "Creating the runner for #{b.key}" }
+          pool[b.key] = Runner.new(b, max_ts).tap(&:start)
+        end
+
+        while running
+          begin
+            # Stop disabled blockchains runners first.
+            (@runner_pool.keys - ::Blockchain.active.pluck(:key)).each do |b_key|
+              logger.warn { "Stopping the runner for #{b_key} (blockchain is not active anymore)" }
+              @runner_pool.delete(b_key).stop
+            end
+
+            # Recreate active blockchain runners by comparing runner &
+            # maximum blockchain & currencies updated_at timestamp.
+            ::Blockchain.active.each do |b|
+              max_ts = [b.currencies.maximum(:updated_at), b.updated_at].compact.max.to_i
+
+              if @runner_pool[b.key].blank?
+                logger.warn { "Starting the new runner for #{b.key} (no runner found in pool)" }
+                @runner_pool[b.key] = Runner.new(b, max_ts).tap(&:start)
+              elsif @runner_pool[b.key].ts < max_ts
+                logger.warn { "Recreating a runner for #{b.key} (#{Time.at(@runner_pool[b.key].ts)} < #{Time.at(max_ts)})" }
+                @runner_pool.delete(b.key).stop
+                @runner_pool[b.key] = Runner.new(b, max_ts).tap(&:start)
+              else
+                logger.warn { "The runner for #{b.key} is up to date (#{Time.at(@runner_pool[b.key].ts)} >= #{Time.at(max_ts)})" }
+              end
+            end
+
+            logger.info { "Current runners timestamps:" }
+            logger.info do
+              @runner_pool.transform_values(&:ts)
+            end
+
+            # Check for blockchain config changes in 30 seconds.
+            sleep 30
+
+          rescue StandardError => e
+            raise e if is_db_connection_error?(e)
+
+            report_exception(e)
+            Rails.logger.warn { "Error: #{e}. Sleeping for 10 seconds" }
+            sleep(10)
+          end
+        end
+      end
+
+      def stop
+        @running = false
+        @runner_pool.each { |_bc_key, runner| runner.stop }
+      end
+    end
+  end
+end

app/workers/daemons/cron_job.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# K-line point is represented as array of 5 numbers:
+# [timestamp, open_price, max_price, min_price, last_price, period_volume]
+
+module Workers
+  module Daemons
+    class CronJob < Base
+      JOBS = [Jobs::Cron::KLine, Jobs::Cron::Ticker, Jobs::Cron::StatsMemberPnl, Jobs::Cron::AML, Jobs::Cron::Refund, Jobs::Cron::WalletBalances, Jobs::Cron::ReferralBonus].freeze
+
+      def run
+        JOBS.map { |j| Thread.new { process(j) } }.map(&:join)
+      end
+
+      def process(service)
+        service.process while running
+      end
+    end
+  end
+end

app/workers/daemons/deposit.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Workers
+  module Daemons
+    class Deposit < Base
+      self.sleep_time = 60
+
+      def process
+        # Process deposits with `processing` state each minute
+        ::Deposit.processing.each do |deposit|
+          Rails.logger.info { "Starting processing coin deposit with id: #{deposit.id}." }
+
+          wallet = Wallet.deposit_wallet(deposit.currency_id)
+          unless wallet
+            Rails.logger.warn { "Can't find active deposit wallet for currency with code: #{deposit.currency_id}."}
+            next
+          end
+          service = WalletService.new(wallet)
+          # Check if adapter has prepare_deposit_collection! implementation
+          if service.adapter.class.instance_methods(false).include?(:prepare_deposit_collection!)
+            begin
+              # Process fee collection for tokens
+              collect_fee(deposit)
+              # Will be processed after fee collection
+              next if deposit.fee_processing?
+            rescue StandardError => e
+              Rails.logger.error { "Failed to collect deposit fee #{deposit.id}. See exception details below." }
+              report_exception(e)
+              raise e if is_db_connection_error?(e)
+
+              next
+            end
+          end
+
+          process_deposit(deposit)
+        end
+
+        # Process deposits with `fee_processing` state that already collected fees for collection
+        ::Deposit.fee_processing.where('updated_at < ?', 5.minute.ago).each do |deposit|
+          Rails.logger.info { "Starting processing token deposit with id: #{deposit.id}." }
+
+          process_deposit(deposit)
+        end
+      end
+
+      def process_deposit(deposit)
+        if deposit.spread.blank?
+          deposit.spread_between_wallets!
+          Rails.logger.warn { "The deposit was spreaded in the next way: #{deposit.spread}"}
+        end
+
+        wallet = Wallet.deposit_wallet(deposit.currency_id)
+        service = WalletService.new(wallet)
+
+        transactions = service.collect_deposit!(deposit, deposit.spread_to_transactions)
+
+        if transactions.present?
+          # Save txids in deposit spread.
+          deposit.update!(spread: transactions.map(&:as_json))
+
+          Rails.logger.warn { "The API accepted deposit collection and assigned transaction ID: #{transactions.map(&:as_json)}." }
+
+          deposit.dispatch!
+        else
+          deposit.skip!
+          "Skipped deposit with txid: #{deposit.txid} with amount: #{deposit.amount}"\
+          " to #{deposit.address}"
+        end
+      rescue StandardError => e
+        Rails.logger.error { "Failed to collect deposit #{deposit.id}. See exception details below." }
+        report_exception(e)
+
+        raise e if is_db_connection_error?(e)
+      end
+
+      def collect_fee(deposit)
+        if deposit.spread.blank?
+          deposit.spread_between_wallets!
+          Rails.logger.warn { "The deposit was spreaded in the next way: #{deposit.spread}"}
+        end
+
+        fee_wallet = Wallet.active.fee.find_by(blockchain_key: deposit.currency.blockchain_key)
+        unless fee_wallet
+          Rails.logger.warn { "Can't find active fee wallet for currency with code: #{deposit.currency_id}." }
+          return
+        end
+
+        transactions = WalletService.new(fee_wallet).deposit_collection_fees!(deposit, deposit.spread_to_transactions)
+        deposit.fee_process! if transactions.present?
+        Rails.logger.warn { "The API accepted token deposit collection fee and assigned transaction ID: #{transactions.map(&:as_json)}." }
+      end
+    end
+  end
+end

app/workers/daemons/upstream.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Workers
+  module Daemons
+    class Upstream < Base
+      def run
+        Engine.all.map { |e| Thread.new { process(e) } }.map(&:join)
+      end
+
+      def process(engine)
+        EM.synchrony do
+          upstream = Peatio::Upstream.registry[engine.driver]
+          engine.markets.each do |market|
+            target = if market.data.present? && market.data['target'].present?
+                       market.data['target']
+                     else
+                       market.id
+                     end
+
+            configs = engine.data.merge('source' => market.id, 'amqp' => ::AMQP::Queue, 'target' => target)
+
+            upstream.new(configs).ws_connect
+            Rails.logger.info "Upstream with driver #{engine.driver} for #{market.id} started"
+          rescue StandardError => e
+            report_exception(e)
+            next
+          end
+        rescue Peatio::AdapterRegistry::NotRegisteredAdapterError => e
+          report_exception(e)
+        end
+      end
+
+      def stop
+        puts 'Shutting down'
+        @shutdown = true
+        exit(42)
+      end
+    end
+  end
+end