SwtTt29
/
JDY2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

实现自动同步功能：数据库新增数据后简道云自动同步更新

Integration
SwTt29 3 months ago
commit
0b9e954573
9 changed files with 1370 additions and 0 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 26
      .gitignore
  2. 152
      README.md
  3. 134
      index.js
  4. 431
      package-lock.json
  5. 17
      package.json
  6. 99
      src/config/config.js
  7. 232
      src/services/databaseService.js
  8. 233
      src/services/jiandaoyunService.js
  9. 46
      test-incremental.js

26
.gitignore
View File

@ -0,0 +1,26 @@
# Node.js dependencies
node_modules/
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
# Editor directories and files
.idea
.vscode
*.swp
*.swo
*~
.DS_Store
# Environment variables
.env
.env.local
.env.*.local
# Build outputs
dist/
build/

152
README.md
View File

@ -0,0 +1,152 @@
# 数据同步服务
## 简介
这是一个用于将数据库中的用户数据同步到简道云表单的服务。
## 功能特性
1. **数据同步**：将数据库中的用户数据同步到简道云表单
2. **重复数据检查**：跳过简道云表单中已存在的电话号码数据
3. **增量同步**：只同步新增/未同步的数据（默认启用）
4. **全量同步**：同步所有数据
5. **自动同步**：定期自动检查并同步数据库中的新增数据
## 配置选项
### 数据库配置
在 `src/config/config.js` 中配置数据库连接信息：
```javascript
db: {
host: '数据库主机地址',
port: 数据库端口号,
user: '数据库用户名',
password: '数据库密码',
database: '数据库名称'
}
```
### 同步配置
在 `src/config/config.js` 中配置同步选项：
```javascript
sync: {
// 增量同步模式：true - 只同步新增/未同步数据，false - 同步所有数据
incremental: true,
// 同步状态字段
statusField: 'sync_status',
// 最后同步时间字段
timeField: 'last_sync_time',
// 已同步状态值
syncedValue: 1,
// 未同步状态值
unsyncedValue: 0,
// 自动同步配置
autoSync: {
// 是否启用自动同步
enabled: true,
// 同步间隔时间（单位：分钟）
interval: 5
}
}
```
### 简道云配置
在 `src/config/config.js` 中配置简道云API信息：
```javascript
jiandaoyun: {
appId: '简道云应用ID',
entryId: '简道云表单ID',
apiKey: '简道云API密钥'
}
```
### 字段映射
在 `src/config/config.js` 中配置数据库字段与简道云表单字段的映射关系：
```javascript
fieldMapping: {
'userId': '简道云字段ID',
'company': '简道云字段ID',
'name': '简道云字段ID',
'phoneNumber': '简道云字段ID',
'type': '简道云字段ID',
'city': '简道云字段ID'
// 其他字段映射...
}
```
## 使用方法
### 安装依赖
```bash
npm install
```
### 运行服务
#### 启动自动同步服务
```bash
node index.js
```
这将启动自动同步服务，每隔5分钟（可在配置中修改）自动检查并同步数据库中的新增数据。
#### 执行一次全量同步
```bash
node index.js --full-sync
```
这将执行一次全量同步，同步数据库中的所有数据到简道云表单。
#### 执行一次增量同步
```bash
node index.js --incremental-sync
```
这将执行一次增量同步，只同步数据库中未同步的数据到简道云表单。
#### 测试数据库连接
```bash
node index.js --test-db
```
这将测试数据库连接是否正常。
#### 测试简道云API连接
```bash
node index.js --test-jiandao
```
这将测试简道云API连接是否正常。
#### 测试所有连接
```bash
node index.js --test-all
```
这将测试数据库连接和简道云API连接是否正常。
## 注意事项
1. 确保数据库中的用户表包含 `sync_status` 和 `last_sync_time` 字段，用于标记同步状态和最后同步时间
2. 确保简道云表单中已创建对应的字段，并在配置文件中正确配置字段映射关系
3. 自动同步服务需要保持运行状态才能定期执行同步
4. 重复的电话号码数据会被跳过，不会重复写入简道云表单
## 终止服务
要终止自动同步服务，可以在终端中按 `Ctrl + C` 键。

134
index.js
View File

@ -0,0 +1,134 @@
// 主程序 - 数据同步服务
const databaseService = require('./src/services/databaseService');
const jiandaoyunService = require('./src/services/jiandaoyunService');
const config = require('./src/config/config');
async function syncData() {
console.log('===== 开始数据同步 =====');
try {
// 1. 连接数据库
await databaseService.connect();
// 2. 查询所有需要同步的数据
console.log('正在查询数据库数据...');
const syncData = await databaseService.getAllSyncData();
console.log(`共查询到 ${syncData.length} 条需要同步的数据`);
if (syncData.length === 0) {
console.log('没有需要同步的数据');
return;
}
// 3. 批量提交数据到简道云
console.log('开始向简道云提交数据...');
const results = await jiandaoyunService.batchSubmitData(syncData);
// 4. 更新同步状态
console.log('\n更新同步状态...');
for (let i = 0; i < results.length; i++) {
const result = results[i];
const data = syncData[i];
if (result.success && !result.skipped) {
// 同步成功且未被跳过，更新状态为已同步
await databaseService.updateSyncStatus(data.userId, true);
}
}
// 5. 统计结果
const successCount = results.filter(result => result.success).length;
const failCount = results.filter(result => !result.success).length;
console.log(`\n===== 数据同步完成 =====`);
console.log(`成功提交: ${successCount} 条`);
console.log(`失败提交: ${failCount} 条`);
// 6. 输出失败详情
if (failCount > 0) {
console.log(`\n失败详情:`);
results.forEach((result, index) => {
if (!result.success) {
console.log(`第 ${index + 1} 条数据失败: ${result.error}`);
}
});
}
} catch (error) {
console.error('数据同步过程中发生错误:', error.message);
} finally {
// 断开数据库连接
await databaseService.disconnect();
}
}
// 测试数据库连接
async function testDatabase() {
console.log('===== 测试数据库连接 =====');
return await databaseService.testDatabaseConnection();
}
// 测试简道云API连接
async function testJiandaoyun() {
console.log('\n===== 测试简道云API连接 =====');
return await jiandaoyunService.testApiConnection();
}
// 主函数
async function main() {
// 解析命令行参数
const args = process.argv.slice(2);
if (args.includes('--test-db')) {
// 仅测试数据库连接
await testDatabase();
} else if (args.includes('--test-jiandao')) {
// 仅测试简道云API连接
await testJiandaoyun();
} else if (args.includes('--test-all')) {
// 测试所有连接
await testDatabase();
await testJiandaoyun();
} else {
// 检查是否需要临时切换同步模式
if (args.includes('--full-sync')) {
console.log('临时切换到全量同步模式');
config.sync.incremental = false;
// 仅执行一次全量同步
await syncData();
} else if (args.includes('--incremental-sync')) {
console.log('临时切换到增量同步模式');
config.sync.incremental = true;
// 仅执行一次增量同步
await syncData();
} else {
// 启动自动同步服务
await startAutoSync();
}
}
}
// 自动同步函数
async function startAutoSync() {
console.log('===== 启动自动同步服务 =====');
console.log(`自动同步间隔时间：${config.sync.autoSync.interval}分钟`);
// 立即执行一次同步
await syncData();
// 设置定时器，定期执行同步
const intervalMs = config.sync.autoSync.interval * 60 * 1000;
setInterval(async () => {
console.log(`\n===== 自动执行数据同步 =====`);
console.log(`当前时间：${new Date().toLocaleString()}`);
await syncData();
}, intervalMs);
}
// 执行主函数
main().catch(error => {
console.error('程序执行失败:', error.message);
process.exit(1);
});

431
package-lock.json
View File

@ -0,0 +1,431 @@
{
"name": "jx",
"version": "1.0.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "jx",
"version": "1.0.0",
"license": "ISC",
"dependencies": {
"axios": "^1.13.2",
"dotenv": "^17.2.3",
"mysql2": "^3.15.3"
}
},
"node_modules/asynckit": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
"integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
"license": "MIT"
},
"node_modules/aws-ssl-profiles": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/aws-ssl-profiles/-/aws-ssl-profiles-1.1.2.tgz",
"integrity": "sha512-NZKeq9AfyQvEeNlN0zSYAaWrmBffJh3IELMZfRpJVWgrpEbtEpnjvzqBPf+mxoI287JohRDoa+/nsfqqiZmF6g==",
"license": "MIT",
"engines": {
"node": ">= 6.0.0"
}
},
"node_modules/axios": {
"version": "1.13.2",
"resolved": "https://registry.npmjs.org/axios/-/axios-1.13.2.tgz",
"integrity": "sha512-VPk9ebNqPcy5lRGuSlKx752IlDatOjT9paPlm8A7yOuW2Fbvp4X3JznJtT4f0GzGLLiWE9W8onz51SqLYwzGaA==",
"license": "MIT",
"dependencies": {
"follow-redirects": "^1.15.6",
"form-data": "^4.0.4",
"proxy-from-env": "^1.1.0"
}
},
"node_modules/call-bind-apply-helpers": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz",
"integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==",
"license": "MIT",
"dependencies": {
"es-errors": "^1.3.0",
"function-bind": "^1.1.2"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/combined-stream": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
"integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
"license": "MIT",
"dependencies": {
"delayed-stream": "~1.0.0"
},
"engines": {
"node": ">= 0.8"
}
},
"node_modules/delayed-stream": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
"integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
"license": "MIT",
"engines": {
"node": ">=0.4.0"
}
},
"node_modules/denque": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
"integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
"license": "Apache-2.0",
"engines": {
"node": ">=0.10"
}
},
"node_modules/dotenv": {
"version": "17.2.3",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz",
"integrity": "sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==",
"license": "BSD-2-Clause",
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://dotenvx.com"
}
},
"node_modules/dunder-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
"integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==",
"license": "MIT",
"dependencies": {
"call-bind-apply-helpers": "^1.0.1",
"es-errors": "^1.3.0",
"gopd": "^1.2.0"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/es-define-property": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
"integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==",
"license": "MIT",
"engines": {
"node": ">= 0.4"
}
},
"node_modules/es-errors": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz",
"integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==",
"license": "MIT",
"engines": {
"node": ">= 0.4"
}
},
"node_modules/es-object-atoms": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz",
"integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==",
"license": "MIT",
"dependencies": {
"es-errors": "^1.3.0"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/es-set-tostringtag": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz",
"integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==",
"license": "MIT",
"dependencies": {
"es-errors": "^1.3.0",
"get-intrinsic": "^1.2.6",
"has-tostringtag": "^1.0.2",
"hasown": "^2.0.2"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/follow-redirects": {
"version": "1.15.11",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz",
"integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==",
"funding": [
{
"type": "individual",
"url": "https://github.com/sponsors/RubenVerborgh"
}
],
"license": "MIT",
"engines": {
"node": ">=4.0"
},
"peerDependenciesMeta": {
"debug": {
"optional": true
}
}
},
"node_modules/form-data": {
"version": "4.0.5",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz",
"integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==",
"license": "MIT",
"dependencies": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.8",
"es-set-tostringtag": "^2.1.0",
"hasown": "^2.0.2",
"mime-types": "^2.1.12"
},
"engines": {
"node": ">= 6"
}
},
"node_modules/function-bind": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
"integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==",
"license": "MIT",
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/generate-function": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.3.1.tgz",
"integrity": "sha512-eeB5GfMNeevm/GRYq20ShmsaGcmI81kIX2K9XQx5miC8KdHaC6Jm0qQ8ZNeGOi7wYB8OsdxKs+Y2oVuTFuVwKQ==",
"license": "MIT",
"dependencies": {
"is-property": "^1.0.2"
}
},
"node_modules/get-intrinsic": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz",
"integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==",
"license": "MIT",
"dependencies": {
"call-bind-apply-helpers": "^1.0.2",
"es-define-property": "^1.0.1",
"es-errors": "^1.3.0",
"es-object-atoms": "^1.1.1",
"function-bind": "^1.1.2",
"get-proto": "^1.0.1",
"gopd": "^1.2.0",
"has-symbols": "^1.1.0",
"hasown": "^2.0.2",
"math-intrinsics": "^1.1.0"
},
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/get-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz",
"integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==",
"license": "MIT",
"dependencies": {
"dunder-proto": "^1.0.1",
"es-object-atoms": "^1.0.0"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/gopd": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
"integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==",
"license": "MIT",
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/has-symbols": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz",
"integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==",
"license": "MIT",
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/has-tostringtag": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz",
"integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==",
"license": "MIT",
"dependencies": {
"has-symbols": "^1.0.3"
},
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/hasown": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz",
"integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==",
"license": "MIT",
"dependencies": {
"function-bind": "^1.1.2"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/iconv-lite": {
"version": "0.7.0",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.0.tgz",
"integrity": "sha512-cf6L2Ds3h57VVmkZe+Pn+5APsT7FpqJtEhhieDCvrE2MK5Qk9MyffgQyuxQTm6BChfeZNtcOLHp9IcWRVcIcBQ==",
"license": "MIT",
"dependencies": {
"safer-buffer": ">= 2.1.2 < 3.0.0"
},
"engines": {
"node": ">=0.10.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/is-property": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz",
"integrity": "sha512-Ks/IoX00TtClbGQr4TWXemAnktAQvYB7HzcCxDGqEZU6oCmb2INHuOoKxbtR+HFkmYWBKv/dOZtGRiAjDhj92g==",
"license": "MIT"
},
"node_modules/long": {
"version": "5.3.2",
"resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz",
"integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==",
"license": "Apache-2.0"
},
"node_modules/lru.min": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/lru.min/-/lru.min-1.1.3.tgz",
"integrity": "sha512-Lkk/vx6ak3rYkRR0Nhu4lFUT2VDnQSxBe8Hbl7f36358p6ow8Bnvr8lrLt98H8J1aGxfhbX4Fs5tYg2+FTwr5Q==",
"license": "MIT",
"engines": {
"bun": ">=1.0.0",
"deno": ">=1.30.0",
"node": ">=8.0.0"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/wellwelwel"
}
},
"node_modules/math-intrinsics": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
"integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==",
"license": "MIT",
"engines": {
"node": ">= 0.4"
}
},
"node_modules/mime-db": {
"version": "1.52.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
"integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
"license": "MIT",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/mime-types": {
"version": "2.1.35",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
"integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
"license": "MIT",
"dependencies": {
"mime-db": "1.52.0"
},
"engines": {
"node": ">= 0.6"
}
},
"node_modules/mysql2": {
"version": "3.15.3",
"resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.15.3.tgz",
"integrity": "sha512-FBrGau0IXmuqg4haEZRBfHNWB5mUARw6hNwPDXXGg0XzVJ50mr/9hb267lvpVMnhZ1FON3qNd4Xfcez1rbFwSg==",
"license": "MIT",
"dependencies": {
"aws-ssl-profiles": "^1.1.1",
"denque": "^2.1.0",
"generate-function": "^2.3.1",
"iconv-lite": "^0.7.0",
"long": "^5.2.1",
"lru.min": "^1.0.0",
"named-placeholders": "^1.1.3",
"seq-queue": "^0.0.5",
"sqlstring": "^2.3.2"
},
"engines": {
"node": ">= 8.0"
}
},
"node_modules/named-placeholders": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/named-placeholders/-/named-placeholders-1.1.4.tgz",
"integrity": "sha512-/qfG0Kk/bLJIvej4FcPQ2KYUJP8iQdU1CTxysNb/U2wUNb+/4K485yeio8iNoiwfqJnsTInXoRPTza0dZWHVJQ==",
"license": "MIT",
"dependencies": {
"lru.min": "^1.1.0"
},
"engines": {
"node": ">=8.0.0"
}
},
"node_modules/proxy-from-env": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
"integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==",
"license": "MIT"
},
"node_modules/safer-buffer": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
"license": "MIT"
},
"node_modules/seq-queue": {
"version": "0.0.5",
"resolved": "https://registry.npmjs.org/seq-queue/-/seq-queue-0.0.5.tgz",
"integrity": "sha512-hr3Wtp/GZIc/6DAGPDcV4/9WoZhjrkXsi5B/07QgX8tsdc6ilr7BFM6PM6rbdAX1kFSDYeZGLipIZZKyQP0O5Q=="
},
"node_modules/sqlstring": {
"version": "2.3.3",
"resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.3.tgz",
"integrity": "sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg==",
"license": "MIT",
"engines": {
"node": ">= 0.6"
}
}
}
}

17
package.json
View File

@ -0,0 +1,17 @@
{
"name": "jx",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"keywords": [],
"author": "",
"license": "ISC",
"dependencies": {
"axios": "^1.13.2",
"dotenv": "^17.2.3",
"mysql2": "^3.15.3"
}
}

99
src/config/config.js
View File

@ -0,0 +1,99 @@
// 数据库连接配置
require('dotenv').config();
module.exports = {
db: {
host: '1.95.162.61',
port: 3306,
user: 'root',
password: 'schl@2025',
database: 'wechat_app'
},
// 数据库表结构配置
tables: {
users: {
name: 'users',
fields: {
id: 'id',
userId: 'userId',
company: 'company',
name: 'name',
phoneNumber: 'phoneNumber',
type: 'type',
city: 'city'
}
},
cartItems: {
name: 'cart_items',
fields: {
userId: 'userId',
productName: 'productName',
specification: 'specification',
quantity: 'quantity',
yolk: 'yolk'
}
},
products: {
name: 'products',
fields: {
sellerId: 'sellerId',
productName: 'productName',
specification: 'specification',
quantity: 'quantity',
yolk: 'yolk'
}
}
},
// 简道云配置
jiandaoyun: {
appId: '684bd1da61702bed58d15d13',
entryId: '693644f891a2fb633d1c0e8f',
apiKey: 'JgTzhmiwlwzz4LWl4dJ4FGZ6yr3VqxoW',
apiUrl: 'https://api.jiandaoyun.com/api/v1'
},
// 字段映射关系
fieldMapping: {
'userId': '_widget_1765164283327',
'company': '_widget_1765164283326',
'name': '_widget_1765164283341',
'phoneNumber': '_widget_1765164283342',
'type': '_widget_1765171392031',
'city': '_widget_1765164283330',
// cart_items表映射 (buyer相关)
'productName-buyer': '_widget_1765164283332',
'specification-buyer': '_widget_1765164283336',
'quantity-buyer': '_widget_1765164283337',
'grossWeight-buyer': '_widget_1765164283338',
'yolk-buyer': '_widget_1765164283339',
// products表映射 (sell相关)
'productName-sell': '_widget_1765178808518',
'specification-sell': '_widget_1765178808519',
'quantity-sell': '_widget_1765178808520',
'grossWeight-sell': '_widget_1765178808521',
'yolk-sell': '_widget_1765178808522'
},
// 同步配置
sync: {
// 增量同步模式：true - 只同步新增/未同步数据，false - 同步所有数据
incremental: true,
// 同步状态字段
statusField: 'sync_status',
// 最后同步时间字段
timeField: 'last_sync_time',
// 已同步状态值
syncedValue: 1,
// 未同步状态值
unsyncedValue: 0,
// 自动同步配置
autoSync: {
// 是否启用自动同步
enabled: true,
// 同步间隔时间（单位：分钟）
interval: 5
}
}
};

232
src/services/databaseService.js
View File

@ -0,0 +1,232 @@
// 数据库服务
const mysql = require('mysql2/promise');
const config = require('../config/config');
class DatabaseService {
constructor() {
this.connection = null;
}
// 连接数据库
async connect() {
try {
this.connection = await mysql.createConnection({
host: config.db.host,
port: config.db.port,
user: config.db.user,
password: config.db.password,
database: config.db.database
});
console.log('数据库连接成功');
} catch (error) {
console.error('数据库连接失败:', error.message);
throw error;
}
}
// 断开数据库连接
async disconnect() {
if (this.connection) {
await this.connection.end();
console.log('数据库连接已断开');
}
}
// 查询需要同步的数据
async getAllSyncData() {
try {
// 根据配置决定是查询所有数据还是仅未同步数据
let usersQuery = `SELECT ${config.tables.users.fields.id},
${config.tables.users.fields.userId},
${config.tables.users.fields.company},
${config.tables.users.fields.name},
${config.tables.users.fields.phoneNumber},
${config.tables.users.fields.type},
${config.tables.users.fields.city}
FROM ${config.tables.users.name}`;
// 如果是增量同步，只查询未同步的数据
if (config.sync.incremental) {
usersQuery += ` WHERE ${config.sync.statusField} = ${config.sync.unsyncedValue}`;
console.log('启用增量同步模式，只查询未同步的数据');
} else {
console.log('启用全量同步模式，查询所有数据');
}
console.log('执行查询:', usersQuery);
const [users] = await this.connection.execute(usersQuery);
console.log(`查询结果: 共找到 ${users.length} 条需要同步的用户数据`);
const syncData = [];
// 为每个用户查询对应的cart_items和products数据
for (const user of users) {
const userId = user[config.tables.users.fields.userId];
// 查询cart_items表（buyer数据）
const [cartItems] = await this.connection.execute(
`SELECT ${config.tables.cartItems.fields.productName},
${config.tables.cartItems.fields.specification},
${config.tables.cartItems.fields.quantity},
${config.tables.cartItems.fields.yolk}
FROM ${config.tables.cartItems.name}
WHERE ${config.tables.cartItems.fields.userId} = ?`,
[userId]
);
// 查询products表（sell数据）
const [products] = await this.connection.execute(
`SELECT ${config.tables.products.fields.productName},
${config.tables.products.fields.specification},
${config.tables.products.fields.quantity},
${config.tables.products.fields.yolk}
FROM ${config.tables.products.name}
WHERE ${config.tables.products.fields.sellerId} = ?`,
[userId]
);
syncData.push({
user,
cartItems,
products,
userId: userId // 保存用户ID，用于同步后更新状态
});
}
return syncData;
} catch (error) {
console.error('查询数据失败:', error.message);
throw error;
}
}
// 获取一条测试数据用于API测试
async getTestData() {
try {
// 获取一条用户数据
const [users] = await this.connection.execute(
`SELECT ${config.tables.users.fields.id},
${config.tables.users.fields.userId},
${config.tables.users.fields.company},
${config.tables.users.fields.name},
${config.tables.users.fields.phoneNumber},
${config.tables.users.fields.type},
${config.tables.users.fields.city}
FROM ${config.tables.users.name} LIMIT 1`
);
if (users.length === 0) {
console.error('没有找到用户数据');
return null;
}
const user = users[0];
const userId = user[config.tables.users.fields.id];
// 获取该用户的购物车数据
const [cartItems] = await this.connection.execute(
`SELECT ${config.tables.cartItems.fields.productName},
${config.tables.cartItems.fields.specification},
${config.tables.cartItems.fields.quantity},
${config.tables.cartItems.fields.yolk}
FROM ${config.tables.cartItems.name}
WHERE ${config.tables.cartItems.fields.userId} = ?`,
[userId]
);
// 获取该用户的产品数据
const [products] = await this.connection.execute(
`SELECT ${config.tables.products.fields.productName},
${config.tables.products.fields.specification},
${config.tables.products.fields.quantity},
${config.tables.products.fields.yolk}
FROM ${config.tables.products.name}
WHERE ${config.tables.products.fields.sellerId} = ?`,
[userId]
);
return {
user,
cartItems,
products
};
} catch (error) {
console.error('获取测试数据失败:', error.message);
throw error;
}
}
// 更新数据同步状态
async updateSyncStatus(userId, synced) {
try {
const statusValue = synced ? config.sync.syncedValue : config.sync.unsyncedValue;
const now = new Date();
await this.connection.execute(
`UPDATE ${config.tables.users.name}
SET ${config.sync.statusField} = ?, ${config.sync.timeField} = ?
WHERE ${config.tables.users.fields.userId} = ?`,
[statusValue, now, userId]
);
console.log(`用户 ${userId} 的同步状态已更新为: ${synced ? '已同步' : '未同步'}`);
} catch (error) {
console.error(`更新用户 ${userId} 同步状态失败:`, error.message);
throw error;
}
}
// 测试数据库连接和表结构
async testDatabaseConnection() {
try {
// 测试连接
await this.connect();
// 查询表结构
const tables = [config.tables.users.name, config.tables.cartItems.name, config.tables.products.name];
for (const table of tables) {
console.log(`\n--- 表 ${table} 结构 ---`);
const [columns] = await this.connection.execute(`DESCRIBE ${table}`);
columns.forEach(column => {
console.log(`${column.Field}: ${column.Type} ${column.Null === 'YES' ? '(允许为空)' : '(不允许为空)'} ${column.Key === 'PRI' ? '(主键)' : ''}`);
});
}
// 查询示例数据
console.log(`\n--- 示例数据 ---`);
const [usersSample] = await this.connection.execute(`SELECT * FROM ${config.tables.users.name} LIMIT 1`);
if (usersSample.length > 0) {
console.log('用户表示例数据:', usersSample[0]);
const userId = usersSample[0][config.tables.users.fields.id];
const [cartItemsSample] = await this.connection.execute(
`SELECT * FROM ${config.tables.cartItems.name} WHERE ${config.tables.cartItems.fields.userId} = ? LIMIT 1`,
[userId]
);
if (cartItemsSample.length > 0) {
console.log('购物车表示例数据:', cartItemsSample[0]);
}
const [productsSample] = await this.connection.execute(
`SELECT * FROM ${config.tables.products.name} WHERE ${config.tables.products.fields.sellerId} = ? LIMIT 1`,
[userId]
);
if (productsSample.length > 0) {
console.log('产品表示例数据:', productsSample[0]);
}
}
await this.disconnect();
return true;
} catch (error) {
console.error('数据库测试失败:', error.message);
if (this.connection) {
await this.disconnect();
}
return false;
}
}
}
module.exports = new DatabaseService();

233
src/services/jiandaoyunService.js
View File

@ -0,0 +1,233 @@
// 简道云API服务
const axios = require('axios');
const config = require('../config/config');
class JiandaoyunService {
constructor() {
// 简道云API基础配置
this.baseUrl = 'https://api.jiandaoyun.com';
this.apiKey = config.jiandaoyun.apiKey;
this.appId = config.jiandaoyun.appId;
this.entryId = config.jiandaoyun.entryId;
}
// 提交数据到简道云表单
async submitDataToForm(data) {
try {
console.log('准备提交数据到简道云:', JSON.stringify(data, null, 2));
// 简道云API v1的正确请求格式 - 使用data_create端点
const url = `${this.baseUrl}/api/v1/app/${this.appId}/entry/${this.entryId}/data_create`;
const headers = {
'Content-Type': 'application/json',
'Authorization': `Bearer ${this.apiKey}`
};
// 简道云API v1只需要在请求体中包含data字段
const payload = {
data: data
};
console.log('请求URL:', url);
console.log('请求头:', headers);
console.log('请求体:', JSON.stringify(payload, null, 2));
const response = await axios.post(url, payload, { headers });
console.log('简道云API调用成功:', response.data);
return response.data;
} catch (error) {
console.error('简道云API调用失败:', error.message);
if (error.response) {
console.error('响应状态:', error.response.status);
console.error('响应数据:', JSON.stringify(error.response.data, null, 2));
console.error('响应头:', JSON.stringify(error.response.headers, null, 2));
if (error.config) {
console.error('请求URL:', error.config.url);
console.error('请求头:', JSON.stringify(error.config.headers, null, 2));
console.error('请求数据:', error.config.data);
}
}
throw error;
}
}
// 将数据库数据转换为简道云表单所需格式
transformDataToJiandaoyunFormat(databaseData) {
console.log('开始转换数据:', JSON.stringify(databaseData, null, 2));
const jiandaoyunData = {};
const mapping = config.fieldMapping;
// 转换主表数据
const user = databaseData.user;
console.log('用户数据:', JSON.stringify(user, null, 2));
console.log('字段映射:', JSON.stringify(mapping, null, 2));
// 使用简道云API v1的正确值格式，使用value字段来包装值
jiandaoyunData[mapping.userId] = { value: user.userId || '' };
jiandaoyunData[mapping.company] = { value: user.company || '' };
jiandaoyunData[mapping.name] = { value: user.name || '' };
jiandaoyunData[mapping.phoneNumber] = { value: user.phoneNumber || '' };
jiandaoyunData[mapping.type] = { value: user.type || '' };
jiandaoyunData[mapping.city] = { value: user.city || '' };
// 转换cart_items数据（buyer）
const cartItems = databaseData.cartItems;
console.log('购物车数据:', JSON.stringify(cartItems, null, 2));
if (cartItems.length > 0) {
const firstCartItem = cartItems[0];
jiandaoyunData[mapping['productName-buyer']] = { value: firstCartItem.productName || '' };
jiandaoyunData[mapping['specification-buyer']] = { value: firstCartItem.specification || '' };
jiandaoyunData[mapping['quantity-buyer']] = { value: firstCartItem.quantity || '' };
// 计算毛重：数量 * 规格中的克数
const specification = firstCartItem.specification || '';
const weightPerUnit = parseInt(specification.match(/(\d+)克/)?.[1] || '0');
const quantity = parseInt(firstCartItem.quantity || '0');
const grossWeight = weightPerUnit * quantity;
jiandaoyunData[mapping['grossWeight-buyer']] = { value: grossWeight.toString() };
jiandaoyunData[mapping['yolk-buyer']] = { value: firstCartItem.yolk || '' };
}
// 转换products数据（sell）
const products = databaseData.products;
console.log('产品数据:', JSON.stringify(products, null, 2));
if (products.length > 0) {
const firstProduct = products[0];
jiandaoyunData[mapping['productName-sell']] = { value: firstProduct.productName || '' };
jiandaoyunData[mapping['specification-sell']] = { value: firstProduct.specification || '' };
jiandaoyunData[mapping['quantity-sell']] = { value: firstProduct.quantity || '' };
// 计算毛重：数量 * 规格中的克数
const specification = firstProduct.specification || '';
const weightPerUnit = parseInt(specification.match(/(\d+)克/)?.[1] || '0');
const quantity = parseInt(firstProduct.quantity || '0');
const grossWeight = weightPerUnit * quantity;
jiandaoyunData[mapping['grossWeight-sell']] = { value: grossWeight.toString() };
jiandaoyunData[mapping['yolk-sell']] = { value: firstProduct.yolk || '' };
}
console.log('转换后的数据:', JSON.stringify(jiandaoyunData, null, 2));
return jiandaoyunData;
}
// 查询简道云表单中是否存在指定电话号码的数据
async isPhoneNumberExists(phoneNumber) {
try {
const mapping = config.fieldMapping;
const url = `${this.baseUrl}/api/v1/app/${this.appId}/entry/${this.entryId}/data_list`;
const headers = {
'Content-Type': 'application/json',
'Authorization': `Bearer ${this.apiKey}`
};
// 构建查询条件：电话号码字段等于指定值
const payload = {
filter: {
rel: 'and',
cond: [
{
field: mapping.phoneNumber,
type: 'text',
method: 'eq',
value: phoneNumber
}
]
},
page_size: 1 // 只需要知道是否存在，不需要返回所有结果
};
const response = await axios.post(url, payload, { headers });
// 如果返回的数据数量大于0，则表示该电话号码已存在
return response.data.data.length > 0;
} catch (error) {
console.error('查询电话号码是否存在失败:', error.message);
if (error.response) {
console.error('响应状态:', error.response.status);
console.error('响应数据:', error.response.data);
}
// 发生错误时，为了避免数据丢失，默认返回false，表示不存在该电话号码
return false;
}
}
// 批量提交数据到简道云
async batchSubmitData(dataList) {
const results = [];
for (const data of dataList) {
try {
// 检查电话号码是否已存在
const phoneNumber = data.user.phoneNumber;
const exists = await this.isPhoneNumberExists(phoneNumber);
if (exists) {
console.log(`电话号码 ${phoneNumber} 已存在于简道云表单中，跳过同步`);
results.push({
success: true,
skipped: true,
message: `电话号码 ${phoneNumber} 已存在，跳过同步`,
originalData: data
});
continue;
}
// 转换数据格式
const jiandaoyunData = this.transformDataToJiandaoyunFormat(data);
// 提交数据
const result = await this.submitDataToForm(jiandaoyunData);
results.push({
success: true,
data: result,
originalData: data
});
console.log('数据提交成功:', result);
} catch (error) {
results.push({
success: false,
error: error.message,
originalData: data
});
console.error('数据提交失败:', error.message);
}
}
return results;
}
// 测试简道云API连接
async testApiConnection() {
try {
console.log('===== 测试简道云API连接 =====');
// 使用fieldMapping来构建测试数据，验证字段映射是否正确
const mapping = config.fieldMapping;
const testData = {
[mapping.userId]: { value: '123456' }, // 测试用户ID
[mapping.company]: { value: '测试公司' },
[mapping.name]: { value: '测试联系人' },
[mapping.phoneNumber]: { value: '13800138000' },
[mapping.type]: { value: '零售客户' }, // 测试客户类型
[mapping.city]: { value: '北京' } // 测试地区
};
const response = await this.submitDataToForm(testData);
console.log('简道云API连接成功');
return true;
} catch (error) {
console.error('简道云API连接失败:', error.message);
if (error.response) {
console.error('响应状态:', error.response.status);
console.error('响应数据:', error.response.data);
}
console.error('错误堆栈:', error.stack);
return false;
}
}
}
module.exports = new JiandaoyunService();

46
test-incremental.js
View File

@ -0,0 +1,46 @@
// 测试增量同步功能
const DatabaseService = require('./src/services/databaseService');
const config = require('./src/config/config');
async function testIncrementalSync() {
console.log('===== 测试增量同步功能 =====');
try {
// 连接数据库
await DatabaseService.connect();
// 检查配置
console.log('当前同步配置:', JSON.stringify(config.sync, null, 2));
// 查询需要同步的数据
console.log('查询需要同步的数据...');
const syncData = await DatabaseService.getAllSyncData();
console.log(`\n测试结果:`);
console.log(`增量同步模式: ${config.sync.incremental ? '开启' : '关闭'}`);
console.log(`查询到需要同步的数据条数: ${syncData.length}`);
if (syncData.length > 0) {
console.log('\n前3条数据示例:');
for (let i = 0; i < Math.min(3, syncData.length); i++) {
console.log(`\n用户ID: ${syncData[i].userId}`);
console.log(`公司: ${syncData[i].user.company}`);
console.log(`联系人: ${syncData[i].user.nickName}`);
console.log(`购买商品数量: ${syncData[i].cartItems.length}`);
console.log(`销售商品数量: ${syncData[i].products.length}`);
}
}
} catch (error) {
console.error('测试过程中发生错误:', error.message);
} finally {
// 断开数据库连接
await DatabaseService.disconnect();
}
}
// 执行测试
testIncrementalSync().catch(error => {
console.error('测试执行失败:', error.message);
process.exit(1);
});
Write Preview
Loading…
Cancel
Save