将pg_partman与Rails和HerokuCI一起使用

Question

bounty到期在7天内。该问题的答案有资格获得+ 500声望赏金。Schwern想要引起更多关注对此问题。

在我的Rails 6应用中，我正在创建一个我知道会变大的表。因此，我要使用pg_partman按月对它进行分区。它托管在Heroku上，因此我关注了their instructions。迁移看起来像这样：

class CreateReceipts < ActiveRecord::Migration[6.0]
  def change
    reversible do |dir|
      dir.up do
        execute <<-SQL
          create extension pg_partman;
        SQL
      end
      dir.down do
        execute <<-SQL
          drop extension pg_partman;
        SQL
      end
    end

    create_table(
      :receipts,
      # Partitioning requires the primary key includes the column we're partitioning by.
      primary_key: [:id, :created_at],
      options: 'partition by range (created_at)'
    ) do |t|
      # When using the primary key option, it ignores id: true. Make the ID column manually.
      t.column :id, :bigserial, null: false

      t.references :customer, null: false, foreign_key: true
      t.integer :thing, null: false
      t.text :stuff, null: false

      t.timestamps
    end

    reversible do |dir|
      dir.up do
        execute <<-SQL
          select create_parent('public.receipts', 'created_at', 'native', 'monthly');
        SQL
      end
      dir.down do
        # Dropping receipts undoes all the partitioning, except the template table.
        drop_table(:template_public_receipts)
      end
    end
  end
end

class Receipt < ApplicationRecord
  # The composite primary key is only for partitioning.
  self.primary_key = 'id'

  # Unfortunately, partitioning gets confused if we add another unique index.
  # So we must enforce ID uniqueness in the model.
  validates :id, uniqueness: true
end

主键有些奇怪，但是在本地可以正常使用。 Heroku Postgres具有pg_partman扩展名，因此可以正常生产。

问题是HerokuCI。我正在使用推荐的in-dyno database插件。它没有pg_partman。

-----> Preparing test database
       Running: rake db:schema:load_if_ruby
       db:schema:load_if_ruby completed (6.17s)
       Running: rake db:structure:load_if_sql
        set_config 
       ------------

       (1 row)

       psql:/app/db/structure.sql:16: ERROR:  could not open extension control file "/app/.indyno/vendor/postgresql/share/extension/pg_partman.control": No such file or directory

       rake aborted!
       failed to execute:

我宁愿不必为此仅将完整的数据库附加到CI。将pg_partman分区硬编码到模式中感觉很奇怪，尽管最好使测试尽可能接近生产环境。

是否有其他方法？

Answer 1

我通过使用In-Dyno Postgres在HerokuCI上检测到pg_partman作为db:structure:load_if_sql任务的一部分来解决此问题。

Rake::Task["db:structure:load_if_sql"].enhance [:install_pg_partman]

private def heroku_ci?
  ENV["CI"]
end

private def in_dyno_postgres?
  File.exist?("/app/.indyno/vendor/postgresql")
end

private def install_pg_partman
  system './bin/install-pg-partman'
end

task :install_pg_partman do
  # Heroku In-Dyno Postgres does not have pg_partman.
  if heroku_ci? && in_dyno_postgres?
    puts 'installing pg_partman'
    install_pg_partman
  end
end

和bin/install-pg-partman。

#!/bin/bash

REPO_URL=${REPO_URL='https://github.com/pgpartman/pg_partman.git'}
BUILD_DIR=${BUILD_DIR=tmp}
PG_CONFIG=${PG_CONFIG='/app/.indyno/vendor/postgresql/bin/pg_config'}

if [ ! -f "$PG_CONFIG" ]; then
  echo "Cannot find ${PG_CONFIG}"
  exit 1;
fi

cd tmp
rm -rf pg_partman
git clone ${REPO_URL}
cd pg_partman
make install PG_CONFIG=${PG_CONFIG} NO_BGW=1

有测试。

require 'rails_helper'

RSpec.describe 'rake install_pg_partman' do
  let(:task) { Rake::Task['install_pg_partman'] }

  before do
    # Otherwise if you call the same task twice it will think
    # it's already been done and skip it or re-raise the same exception.
    task.reenable
  end

  shared_context "with CI", temp_env: true do
    before { ENV["CI"] = "true" }
  end

  shared_context "with in-dyno postgres" do
    before {
      allow(File).to receive(:exist?)
        .with("/app/.indyno/vendor/postgresql")
        .and_return(true)
    }
  end

  shared_examples "it does not install" do
    it 'does not install' do
      expect(task).not_to receive(:install_pg_partman)

      task.invoke
    end
  end

  it 'is a prerequisite of db:structure:load_if_sql' do
    expect(
      Rake::Task["db:structure:load_if_sql"].prerequisite_tasks
    ).to include(task)
  end

  context 'no CI, no in-dyno postgres' do
    it_behaves_like 'it does not install'
  end

  context 'when in CI, but no in-dyno postgres' do
    include_context "with CI"

    it_behaves_like 'it does not install'
  end

  context 'with in-dyno postgres, but not in CI' do
    include_context "with in-dyno postgres"

    it_behaves_like 'it does not install'
  end

  context 'when in CI and with in-dyno postgres', temp_env: true do
    include_context "with CI"
    include_context "with in-dyno postgres"

    let(:pg_config_path) { "/does/not/exist/pg_config" }
    before {
      ENV["PG_CONFIG"] = pg_config_path
    }

    it 'tries to install' do
      expect(task.send(:heroku_ci?)).to be_truthy
      expect(task.send(:in_dyno_postgres?)).to be_truthy

      expect {
        task.invoke
      }.to output(/Cannot find #{pg_config_path}/).to_stdout_from_any_process
    end
  end
end