在Keycloak上导出大用户

Question

我想导出拥有近 300,000 个用户的 Keycloak 21.0.2，然后将其导入到另一个系统中的另一个 Keycloak 中。我跑

./kc.sh export --dir exp

。 exp 文件夹生成了近 6000 个名为 myrealm-users-*.json 的文件。但在导入阶段运行

./kc.sh import --dir exp

我发现在某些 json 中我有重复的用户，导致导入阶段失败。

有人有什么建议吗？我该如何解决这个重复的用户？

Answer 1

我测试了 358,000 个用户的导出/导入，但没有错误，也没有重复的导出 json 文件。

这是我的步骤

第0步：准备工作

在 here

在 Windows 上安装 Docker Desktop

在这里安装node.js

node -v
v18.20.4

安装

node.js

依赖项

package.json

{
  "dependencies": {
    "@faker-js/faker": "^9.3.0",
    "axios": "^1.7.9",
    "child_process": "^1.0.2",
    "exceljs": "^4.4.0",
    "fs": "^0.0.1-security",
    "p-limit": "^6.1.0",
    "path": "^0.12.7",
    "qs": "^6.13.1"
  }
}

npm install

save_excel.js

const { faker } = require('@faker-js/faker');
const ExcelJS = require('exceljs');

const USER_COUNT = 358000;
const FILE_NAME = 'Users.xlsx';

// Sets to store used usernames and emails
const usedUsernames = new Set();
const usedEmails = new Set();

// Function to capitalize the first letter of a string
const capitalizeFirstLetter = (string) => {
    return string.charAt(0).toUpperCase() + string.slice(1);
}

// Function to create a random user with a unique username and email
const createRandomUser = (index) => {
    let username, email;

    do {
        username = faker.internet.username().toLowerCase();
        if (usedUsernames.has(username)) {
            username += Math.floor(Math.random() * 10000);
        }
    } while (usedUsernames.has(username));
    usedUsernames.add(username);

    do {
        email = faker.internet.email().toLowerCase();
        if (usedEmails.has(email)) {
            email = email.split('@')[0] + Math.floor(Math.random() * 10000) + '@' + email.split('@')[1];
        }
    } while (usedEmails.has(email));
    usedEmails.add(email);

    return {
        username: username,
        password: `${username}_1234`.toLowerCase(),
        reset: ((index % 2) ? true : false),
        firstName: capitalizeFirstLetter(faker.person.firstName()),
        lastName: capitalizeFirstLetter(faker.person.lastName()),
        email: email
    };
}

// Function to write users to Excel file
const saveUsersToExcel = async (users) => {
    const workbook = new ExcelJS.Workbook();
    const worksheet = workbook.addWorksheet('Users');

    // Define columns
    worksheet.columns = [
        { header: 'Username', key: 'username', width: 30 },
        { header: 'Password', key: 'password', width: 30 },
        { header: 'Reset Password', key: 'reset', width: 15 },
        { header: 'First Name', key: 'firstName', width: 20 },
        { header: 'Last Name', key: 'lastName', width: 20 },
        { header: 'Email', key: 'email', width: 50 }
    ];

    // Sort users by username
    users.sort((a, b) => a.username.localeCompare(b.username));

    // Add users to the worksheet
    worksheet.addRows(users);

    // Save workbook
    await workbook.xlsx.writeFile(FILE_NAME);
    console.log(`Saved ${users.length} users to ${FILE_NAME}`);
}

// Main function to create users and save to Excel
const main = async () => {
    let users = [];

    for (let index = 0; index < USER_COUNT; index++) {
        users.push(createRandomUser(index));
    }

    await saveUsersToExcel(users);
}

main().catch(err => console.error(err));

运行它

node save_excel.js

结果

Users.xlsx

docker-compose.yml

version: '3.6'

services:
  keycloak_web:
    image: quay.io/keycloak/keycloak:21.0.2
    container_name: keycloak_web
    environment:
      KC_DB: postgres
      KC_DB_URL: jdbc:postgresql://keycloakdb:5432/keycloak
      KC_DB_USERNAME: keycloak
      KC_DB_PASSWORD: password

      KC_HOSTNAME: localhost
      KC_HOSTNAME_PORT: 8080
      KC_HOSTNAME_STRICT: false
      KC_HOSTNAME_STRICT_HTTPS: false

      KC_LOG_LEVEL: info
      KC_METRICS_ENABLED: true
      KC_HEALTH_ENABLED: true
      KEYCLOAK_ADMIN: admin
      KEYCLOAK_ADMIN_PASSWORD: admin
    command: start-dev --import-realm
    depends_on:
      - keycloakdb
    ports:
      - 8080:8080
    volumes:
      - ./my-realm-realm.json:/opt/keycloak/data/import/my-realm-realm.json

  keycloakdb:
    image: postgres:15
    volumes:
      - postgres_data:/var/lib/postgresql/data
    environment:
      POSTGRES_DB: keycloak
      POSTGRES_USER: keycloak
      POSTGRES_PASSWORD: password

volumes:
  postgres_data:

详情请看这里

创建

myrealm

主令牌延长1天

create_user.js

(async () => {
    const axios = require('axios');
    const qs = require('qs');
    const ExcelJS = require('exceljs');
    const pLimit = (await import('p-limit')).default;

    // Limit concurrency to 10 requests at a time
    const limit = pLimit(10);

    // Keycloak details
    const client_id = 'admin-cli';
    const user_name = 'admin';
    const pass_word = 'admin';
    const grant_type = 'password';
    const tokenURL = 'http://localhost:8080/realms/master/protocol/openid-connect/token';

    const usersEndpoint = 'http://localhost:8080/admin/realms/myrealm/users';

    // Read from Excel file
    async function readExcelFile() {
        try {
            const workbook = new ExcelJS.Workbook();
            await workbook.xlsx.readFile('Users.xlsx');
            const worksheet = workbook.getWorksheet('Users');
            if (!worksheet) throw new Error("Worksheet 'Users' not found");

            let users = [];
            worksheet.eachRow({ includeEmpty: false }, (row, rowNumber) => {
                if (rowNumber !== 1) { // Skipping header row
                    users.push({
                        username: row.getCell(1).value,
                        firstName: row.getCell(4).value,
                        lastName: row.getCell(5).value,
                        email: row.getCell(6).value
                    });
                }
            });
            return users;
        } catch (error) {
            console.error('Error reading Excel file:', error.message);
            throw error;
        }
    }

    let tokenDetails = { token: null, expiry: 0 }; // Store token and expiry time

    // Function to get a new token
    async function getToken() {
        const data = qs.stringify({
            client_id: client_id,
            username: user_name,
            password: pass_word,
            grant_type: grant_type
        });

        const config = {
            headers: {
                'Content-Type': 'application/x-www-form-urlencoded'
            }
        };

        try {
            const response = await axios.post(tokenURL, data, config);
            const currentTime = Math.floor(Date.now() / 1000); // Current time in seconds
            tokenDetails.token = response.data.access_token;
            tokenDetails.expiry = currentTime + response.data.expires_in; // Calculate expiry time
            return tokenDetails.token;
        } catch (error) {
            console.error('Error fetching token:', error.response ? error.response.data : error.message);
            throw error;
        }
    }

    // Function to ensure the token is valid
    async function ensureToken() {
        const currentTime = Math.floor(Date.now() / 1000);
        if (!tokenDetails.token || currentTime >= tokenDetails.expiry) {
            console.log('Refreshing token...');
            return await getToken();
        }
        return tokenDetails.token;
    }

    async function createUserInKeycloak(user) {
        let attempts = 0;
        const maxAttempts = 3;

        while (attempts < maxAttempts) {
            try {
                const token = await ensureToken(); // Ensure token is valid
                const response = await axios.post(
                    usersEndpoint,
                    {
                        username: user.username,
                        email: user.email,
                        firstName: user.firstName,
                        lastName: user.lastName,
                        requiredActions: [],
                        emailVerified: true,
                        groups: [],
                        enabled: true
                    },
                    {
                        headers: {
                            'Content-Type': 'application/json',
                            'Authorization': `Bearer ${token}`
                        }
                    }
                );

                return response; // Success
            } catch (error) {
                attempts++;
                if (error.response && error.response.status === 401 && attempts < maxAttempts) {
                    console.warn(`Retrying for user ${user.username} (${attempts}/${maxAttempts})...`);
                } else {
                    console.error(
                        `Error creating user ${user.username}:`,
                        error.response ? error.response.data : error.message
                    );
                    return null; // Failure
                }
            }
        }
    }

    async function processBatch(users, successCount) {
        const tasks = users.map(user =>
            limit(async () => {
                const response = await createUserInKeycloak(user);
                if (response) {
                    successCount.count++;
                    console.log(`Current total users added: ${successCount.count} | User created: ${user.username}`);
                }
            })
        );
        await Promise.all(tasks); // Wait for all tasks to complete
    }

    async function main() {
        try {
            const batchSize = 10000;
            const users = await readExcelFile();
            let successCount = { count: 0 }; // Use an object to maintain reference across batches

            for (let i = 0; i < users.length; i += batchSize) {
                const batch = users.slice(i, i + batchSize);
                console.log(`Processing batch ${i / batchSize + 1}...`);
                await processBatch(batch, successCount);
            }

            console.log(`All users have been processed. Total users added: ${successCount.count}`);
        } catch (error) {
            console.error('An error occurred:', error.message);
        }
    }

    main();
})();

运行它

node create_user.js

结果

进入 Keycloak docker 容器

docker exec -it keycloak_web bash

在

tmp2

目录下创建

/bin

目录

cd /opt/keycloak/bin
mkdir tmp2

./kc.sh export --dir tmp2 --users different_files --users-per-file 200

结果

copy-json.js

const { exec } = require('child_process');
const fs = require('fs');
const path = require('path');

const containerName = 'keycloak_web';
const sourcePath = '/opt/keycloak/bin/tmp2';
const destinationPath = './tmp2';

// Ensure the destination directory exists
if (!fs.existsSync(destinationPath)) {
    fs.mkdirSync(destinationPath, { recursive: true });
}

// Function to execute a shell command
function runCommand(command) {
    return new Promise((resolve, reject) => {
        exec(command, (error, stdout, stderr) => {
            if (error) {
                reject(stderr || error.message);
            } else {
                resolve(stdout.trim());
            }
        });
    });
}

(async () => {
    try {
        console.log('Fetching JSON file list from container...');
        const fileList = await runCommand(
            `docker exec ${containerName} sh -c "ls ${sourcePath} | grep \\.json"`
        );
        const files = fileList.split('\n');

        console.log(`Found ${files.length} files. Starting copy...`);

        for (const file of files) {
            const sourceFile = `${sourcePath}/${file}`;
            const destinationFile = path.join(destinationPath, file);

            console.log(`Copying ${file}...`);
            await runCommand(`docker cp ${containerName}:${sourceFile} ${destinationFile}`);
        }

        console.log('All files have been copied successfully.');
    } catch (error) {
        console.error('An error occurred:', error);
    }
})();

从CMD终端

mkdir tmp2
node copy-json.js

结果

计算唯一和重复的用户名

user-count-from-jsons.js

const fs = require('fs');
const path = require('path');

// Directory containing the JSON files
const directoryPath = path.join(__dirname, 'tmp2');

async function countUsernames() {
    try {
        const usernameCounts = new Map();

        // Read all files in the directory
        const files = fs.readdirSync(directoryPath);

        for (const file of files) {
            if (file.startsWith('myrealm-users-') && file.endsWith('.json')) {
                const filePath = path.join(directoryPath, file);

                // Read and parse the JSON file
                const data = JSON.parse(fs.readFileSync(filePath, 'utf-8'));

                // Count occurrences of each username
                for (const user of data.users) {
                    const username = user.username;
                    usernameCounts.set(username, (usernameCounts.get(username) || 0) + 1);
                }
            }
        }

        // Calculate total unique and duplicate usernames
        let duplicateCount = 0;
        for (const [username, count] of usernameCounts) {
            if (count > 1) {
                duplicateCount += (count - 1); // Count duplicates for this username
            }
        }

        console.log(`Total unique usernames: ${usernameCounts.size}`);
        console.log(`Total duplicate usernames: ${duplicateCount}`);
    } catch (error) {
        console.error('An error occurred:', error.message);
    }
}

countUsernames();

结果

终止容器

docker compose down

洁净体积 卷名称取决于您的工作目录

它将清理所有用户

再次运行 docker Keycloak

docker compose up -d

您可以在 Keycloak UI 中检查所有用户是否已清理

再次使用 root 用户访问 Keycloak 容器

docker exec -u root -it keycloak_web /bin/bash

在

tmp2

下创建

/bin

目录

cd /opt/keycloak/bin
mkdir tmp2

从CMD终端

copy-to-container.js

const { exec } = require('child_process');
const fs = require('fs');
const path = require('path');

const containerName = 'keycloak_web'; // Replace with your container name
const destinationPath = '/opt/keycloak/bin/tmp2'; // Destination path inside the container
const sourcePath = './tmp2'; // Source path on the host containing files to copy

// Ensure the source directory exists
if (!fs.existsSync(sourcePath)) {
    console.error(`Source directory does not exist: ${sourcePath}`);
    process.exit(1);
}

// Function to execute a shell command
function runCommand(command) {
    return new Promise((resolve, reject) => {
        exec(command, (error, stdout, stderr) => {
            if (error) {
                reject(stderr || error.message);
            } else {
                resolve(stdout.trim());
            }
        });
    });
}

(async () => {
    try {
        console.log(`Fetching file list from source directory: ${sourcePath}`);
        const files = fs.readdirSync(sourcePath).filter(file => file.endsWith('.json'));

        if (files.length === 0) {
            console.log('No JSON files found in the source directory.');
            return;
        }

        console.log(`Found ${files.length} files. Starting copy to container...`);

        for (const file of files) {
            const sourceFile = path.join(sourcePath, file);
            const containerDestination = `${containerName}:${destinationPath}/${file}`;

            console.log(`Copying ${file} to container...`);
            await runCommand(`docker cp "${sourceFile}" "${containerDestination}"`);
        }

        console.log('All files have been copied successfully.');
    } catch (error) {
        console.error('An error occurred:', error);
    }
})();

运行它

node copy-to-container.js

结果

转到

/bin

目录

docker exec -u root -it keycloak_web /bin/bash
cd /opt/keycloak/bin

导入json

./kc.sh import --dir tmp2

结果

通过node.js检查唯一用户名数

user-count-by-api.js

const axios = require('axios');

// Keycloak details
const keycloakServer = 'http://localhost:8080';
const realm = 'myrealm';
const client_id = 'admin-cli';
const username = 'admin';
const password = 'admin';
const grant_type = 'password';

// Token endpoint
const tokenURL = `${keycloakServer}/realms/master/protocol/openid-connect/token`;

// User count endpoint
const userCountURL = `${keycloakServer}/admin/realms/${realm}/users/count`;

// Function to fetch Keycloak token
async function getToken() {
    try {
        const response = await axios.post(
            tokenURL,
            new URLSearchParams({
                client_id: client_id,
                username: username,
                password: password,
                grant_type: grant_type,
            }),
            {
                headers: {
                    'Content-Type': 'application/x-www-form-urlencoded',
                },
            }
        );

        return response.data.access_token;
    } catch (error) {
        console.error('Error fetching token:', error.response ? error.response.data : error.message);
        throw error;
    }
}

// Function to get user count
async function getUserCount(token) {
    try {
        const response = await axios.get(userCountURL, {
            headers: {
                'Content-Type': 'application/json',
                'Authorization': `Bearer ${token}`,
            },
        });

        return response.data;
    } catch (error) {
        console.error('Error fetching user count:', error.response ? error.response.data : error.message);
        throw error;
    }
}

// Main function
(async () => {
    try {
        console.log('Fetching Keycloak token...');
        const token = await getToken();

        console.log('Fetching user count...');
        const userCount = await getUserCount(token);

        console.log(`Total users in realm "${realm}": ${userCount}`);
    } catch (error) {
        console.error('An error occurred:', error.message);
    }
})();

结果

确认 从 JSON 和 Keycloak UI 中搜索

end of user

在 UI 中，另外两个用户“zula99”在电子邮件中 “zula99”用户名是唯一的。

结论

计算唯一用户名数量

358,000

通过REST API

计算唯一的用户名数量

358,000

是通过扫描jsons

Keycloak 21.0.2版本中导出/导入功能没有重复的用户名