upload-jh #7

Merged
txcjh merged 13 commits from upload-jh into upload 2025-07-17 15:55:28 +08:00
6 changed files with 175 additions and 54 deletions
Showing only changes of commit e7e2c05bcd - Show all commits

View File

@ -3,12 +3,13 @@ import Done from '@/components/user-message.tsx/done';
import Look from '@/components/user-message.tsx/look'; import Look from '@/components/user-message.tsx/look';
import UserName from '@/components/user-message.tsx/userName'; import UserName from '@/components/user-message.tsx/userName';
import { checkAuthStatus } from '@/lib/auth'; import { checkAuthStatus } from '@/lib/auth';
import { getUploadTasks, UploadTask } from '@/lib/db';
import { fetchApi } from '@/lib/server-api-util'; import { fetchApi } from '@/lib/server-api-util';
import { FileUploadItem } from '@/types/upload'; import { FileUploadItem } from '@/types/upload';
import { User } from '@/types/user'; import { User } from '@/types/user';
import { useLocalSearchParams, useRouter } from 'expo-router'; import { useLocalSearchParams, useRouter } from 'expo-router';
import React, { useEffect, useState } from 'react'; import React, { useEffect, useState } from 'react';
import { KeyboardAvoidingView, Platform, ScrollView, StatusBar, View } from 'react-native'; import { KeyboardAvoidingView, Platform, ScrollView, StatusBar, Text, View } from 'react-native';
export type Steps = "userName" | "look" | "choice" | "done"; export type Steps = "userName" | "look" | "choice" | "done";
export default function UserMessage() { export default function UserMessage() {
const router = useRouter(); const router = useRouter();
@ -19,6 +20,7 @@ export default function UserMessage() {
const [fileData, setFileData] = useState<FileUploadItem[]>([]) const [fileData, setFileData] = useState<FileUploadItem[]>([])
const [isLoading, setIsLoading] = useState(false); const [isLoading, setIsLoading] = useState(false);
const [userInfo, setUserInfo] = useState<User | null>(null); const [userInfo, setUserInfo] = useState<User | null>(null);
const [uploadTasks, setUploadTasks] = useState<UploadTask[]>([]); // 新增上传任务状态
const statusBarHeight = StatusBar?.currentHeight ?? 0; const statusBarHeight = StatusBar?.currentHeight ?? 0;
// 获取路由参数 // 获取路由参数
@ -27,6 +29,14 @@ export default function UserMessage() {
useEffect(() => { useEffect(() => {
checkAuthStatus(router); checkAuthStatus(router);
// 设置定时器,每秒查询一次上传进度
const intervalId = setInterval(async () => {
const tasks = await getUploadTasks();
setUploadTasks(tasks);
}, 1000);
return () => clearInterval(intervalId); // 清理定时器
}, []); }, []);
// 获取用户信息 // 获取用户信息
@ -70,6 +80,17 @@ export default function UserMessage() {
keyboardShouldPersistTaps="handled" keyboardShouldPersistTaps="handled"
bounces={false} bounces={false}
> >
{/* 上传进度展示区域 */}
{uploadTasks.length > 0 && (
<View style={{ padding: 10, backgroundColor: '#f0f0f0', borderBottomWidth: 1, borderBottomColor: '#ccc' }}>
<Text style={{ fontWeight: 'bold', marginBottom: 5 }}></Text>
{uploadTasks.map((task) => (
<Text key={task.uri}>
{task.filename}: {task.status} ({task.progress}%)
</Text>
))}
</View>
)}
<View className="h-full" key={steps}> <View className="h-full" key={steps}>
{(() => { {(() => {
const components = { const components = {

View File

@ -1,6 +1,6 @@
import * as BackgroundTask from 'expo-background-task'; import * as BackgroundTask from 'expo-background-task';
import * as TaskManager from 'expo-task-manager'; import * as TaskManager from 'expo-task-manager';
import { initUploadTable } from '../db'; import { getUploadTaskStatus, initUploadTable, insertUploadTask } from '../db';
import { getMediaByDateRange } from './media'; import { getMediaByDateRange } from './media';
import { processAndUploadMedia } from './uploader'; import { processAndUploadMedia } from './uploader';
@ -48,22 +48,37 @@ TaskManager.defineTask(BACKGROUND_UPLOAD_TASK, async () => {
// 串行上传文件 // 串行上传文件
let successCount = 0; let successCount = 0;
let skippedCount = 0; let skippedCount = 0;
let failedCount = 0;
for (const file of media) { for (const file of media) {
try { try {
const existingTask = await getUploadTaskStatus(file.uri);
if (!existingTask) {
// If not in DB, insert as pending
await insertUploadTask(file.uri, file.filename);
} else if (existingTask.status === 'success' || existingTask.status === 'skipped') {
console.log(`File ${file.uri} already ${existingTask.status}, skipping processing.`);
skippedCount++;
continue; // Skip processing if already successful or skipped
}
const result = await processAndUploadMedia(file); const result = await processAndUploadMedia(file);
if (result === null) { if (result === null) {
// 文件已上传,被跳过 // Skipped by processAndUploadMedia (e.g., already uploaded)
skippedCount++; skippedCount++;
} else if (result.originalSuccess) { } else if (result.originalSuccess) {
successCount++; successCount++;
} else {
// Failed
failedCount++;
} }
} catch (e) { } catch (e) {
console.error('Upload failed for', file.uri, e); console.error('Upload failed for', file.uri, e);
failedCount++;
} }
} }
console.log(`Background upload task finished. Successful: ${successCount}, Skipped: ${skippedCount}, Total: ${media.length}`); console.log(`Background upload task finished. Successful: ${successCount}, Skipped: ${skippedCount}, Failed: ${failedCount}, Total: ${media.length}`);
return BackgroundTask.BackgroundTaskResult.Success; return BackgroundTask.BackgroundTaskResult.Success;
} catch (error) { } catch (error) {

View File

@ -3,6 +3,7 @@ import pLimit from 'p-limit';
import { getMediaByDateRange } from './media'; import { getMediaByDateRange } from './media';
import { processAndUploadMedia } from './uploader'; import { processAndUploadMedia } from './uploader';
import { ExtendedAsset } from './types'; import { ExtendedAsset } from './types';
import { insertUploadTask, getUploadTaskStatus } from '../db';
// 设置最大并发数 // 设置最大并发数
const CONCURRENCY_LIMIT = 10; // 同时最多上传10个文件 const CONCURRENCY_LIMIT = 10; // 同时最多上传10个文件
@ -18,7 +19,16 @@ export const triggerManualUpload = async (startDate: Date, endDate: Date) => {
} }
const uploadPromises = media.map((asset: ExtendedAsset) => const uploadPromises = media.map((asset: ExtendedAsset) =>
limit(() => processAndUploadMedia(asset)) limit(async () => {
const existingTask = await getUploadTaskStatus(asset.uri);
if (!existingTask) {
await insertUploadTask(asset.uri, asset.filename);
} else if (existingTask.status === 'success' || existingTask.status === 'skipped') {
console.log(`File ${asset.uri} already ${existingTask.status}, skipping processing.`);
return null; // Skip processing if already successful or skipped
}
return processAndUploadMedia(asset);
})
); );
const results = await Promise.all(uploadPromises); const results = await Promise.all(uploadPromises);

View File

@ -1,28 +1,27 @@
### `lib/background-uploader` 模块概要 此目录包含后台媒体上传的逻辑,包括 API 交互、自动和手动上传过程、媒体库访问以及实用功能。
该模块负责将用户设备中的媒体文件(图片和视频)上传到服务器,并支持后台处理。 **主要组件:**
**核心功能:** - `api.ts`:处理与文件上传相关的 API 调用,例如获取上传 URL、确认上传和添加素材记录。
- `automatic.ts`:实现后台任务的注册和定义,用于自动媒体上传,通常定期运行(例如,每 15 分钟)。它获取最近的媒体并处理它们以上传。
- `manual.ts`:提供在指定日期范围内手动触发媒体上传的功能,并带有并发控制。
- `media.ts`:管理与设备媒体库的交互(使用 `expo-media-library`),包括请求权限、获取带有 EXIF 数据的资产以及按日期范围过滤。
- `types.ts`:定义后台上传模块中使用的 TypeScript 接口和类型,包括 `ExtendedAsset``UploadTask``ConfirmUpload``UploadUrlResponse`
- `uploader.ts`:包含处理和上传单个媒体文件的核心逻辑。这包括处理 HEIC 转换、图像压缩、视频缩略图生成、重复检查以及与上传 API 的交互。它还包括一个 `uploadFileWithProgress` 函数,用于跟踪上传进度。
- `utils.ts`:提供实用功能,例如检查媒体库权限、提取文件扩展名和确定 MIME 类型。
* **媒体选择:** 从设备的媒体库中获取指定日期范围内的照片和视频(见 `media.ts`),同时获取相关的元数据,如 EXIF 和位置信息。 **整体功能:**
* **文件处理:**
* 处理 `HEIC` 格式图片转为 `JPEG` 格式(见 `fileProcessor.ts`)。
* 将图片压缩到标准的宽高和质量(见 `fileProcessor.ts`)。
* 对于视频,提取首帧、压缩后作为缩略图上传(见 `fileProcessor.ts`)。
* **API 交互:**
* 与后端服务器通信,获取用于上传文件的安全临时 URL`api.ts`)。
* 上传完成后与后端确认(见 `api.ts`)。
* 文件及其预览/缩略图上传后,将元数据发送到另一个接口以创建“素材”记录(见 `api.ts`)。
* **上传引擎:**
* 主要上传逻辑位于 `index.ts`,负责整体流程的编排:检查权限、处理文件、调用 API。
* 使用并发限制(`p-limit`),防止同时上传过多文件,提高可靠性和性能。
* 同时处理原始高质量文件和压缩版本(或视频缩略图)的上传。
* **后台任务:**
* 可注册后台任务,定期(如每 15 分钟)自动上传过去 24 小时内的新媒体文件(见 `task.ts`)。
* 即使应用不在前台,也能持续上传文件,提升无缝体验。
* **工具与类型定义:**
* 包含用于检查媒体库权限、获取文件扩展名和 MIME 类型的辅助函数(见 `utils.ts`)。
* 定义了自定义的 `ExtendedAsset` 类型,包含 `exif` 数据和标准的 `MediaLibrary.Asset` 属性(见 `types.ts`)。
* 实际上传文件使用 `XMLHttpRequest`,以支持进度追踪,并通过 Promise 封装(见 `uploader.ts`)。
简而言之,这是一个为移动应用设计的、健壮高效的后台上传系统。 `background-uploader` 模块使应用程序能够:
1. 在后台**自动上传**新创建的媒体文件(照片和视频)。
2. 允许在指定日期范围内**手动上传**媒体文件。
3. 在上传前**处理媒体文件**,包括:
* 将 HEIC 图像转换为 JPEG。
* 压缩图像。
* 为视频生成缩略图。
4. **与后端 API 交互**以管理上传生命周期(获取 URL、上传、确认
5. **管理媒体库权限**并获取带有详细 EXIF 信息的媒体资产。
6. 通过检查本地数据库**防止重复上传**。
7. 提供上传的**进度跟踪**。
该模块通过必要的预处理和错误处理,确保高效、可靠的媒体上传,无论是自动还是按需上传。

View File

@ -6,7 +6,7 @@ import { uploadVideoThumbnail } from '../video-process/videoThumbnail';
import { addMaterial, confirmUpload, getUploadUrl } from './api'; import { addMaterial, confirmUpload, getUploadUrl } from './api';
import { ExtendedAsset } from './types'; import { ExtendedAsset } from './types';
import { checkMediaLibraryPermission, getFileExtension, getMimeType } from './utils'; import { checkMediaLibraryPermission, getFileExtension, getMimeType } from './utils';
import { isFileUploaded, markFileAsUploaded } from '../db'; import { getUploadTaskStatus, updateUploadTaskStatus, updateUploadTaskProgress } from '../db';
// 基础文件上传实现 // 基础文件上传实现
export const uploadFile = async (file: File, uploadUrl: string): Promise<void> => { export const uploadFile = async (file: File, uploadUrl: string): Promise<void> => {
@ -32,13 +32,16 @@ export const uploadFile = async (file: File, uploadUrl: string): Promise<void> =
// 处理单个媒体文件上传的核心逻辑 // 处理单个媒体文件上传的核心逻辑
export const processAndUploadMedia = async (asset: ExtendedAsset) => { export const processAndUploadMedia = async (asset: ExtendedAsset) => {
try { try {
// 1. 文件去重检查 // 1. 文件去重检查 (从数据库获取状态)
const uploaded = await isFileUploaded(asset.uri); const existingTask = await getUploadTaskStatus(asset.uri);
if (uploaded) { if (existingTask && (existingTask.status === 'success' || existingTask.status === 'skipped')) {
console.log('File already uploaded, skipping:', asset.uri); console.log(`File ${asset.uri} already ${existingTask.status}, skipping processing.`);
return null; // 返回 null 表示已上传,调用方可以据此过滤 return null; // 返回 null 表示已上传或已跳过,调用方可以据此过滤
} }
// 标记为正在上传
await updateUploadTaskStatus(asset.uri, 'uploading');
// 2. 检查权限 // 2. 检查权限
const { hasPermission } = await checkMediaLibraryPermission(); const { hasPermission } = await checkMediaLibraryPermission();
if (!hasPermission) { if (!hasPermission) {
@ -97,7 +100,9 @@ export const processAndUploadMedia = async (asset: ExtendedAsset) => {
GPSVersionID: undefined GPSVersionID: undefined
}); });
await uploadFile(fileToUpload, upload_url); await uploadFileWithProgress(fileToUpload, upload_url, (progress) => {
updateUploadTaskProgress(asset.uri, Math.round(progress * 0.5)); // 原始文件占总进度的50%
});
await confirmUpload(file_id); await confirmUpload(file_id);
return { success: true, file_id, filename: fileToUpload.name }; return { success: true, file_id, filename: fileToUpload.name };
@ -121,7 +126,9 @@ export const processAndUploadMedia = async (asset: ExtendedAsset) => {
isCompressed: true isCompressed: true
}); });
await uploadFile(compressedFile, upload_url); await uploadFileWithProgress(compressedFile, upload_url, (progress) => {
updateUploadTaskProgress(asset.uri, 50 + Math.round(progress * 0.5)); // 压缩文件占总进度的后50%
});
await confirmUpload(file_id); await confirmUpload(file_id);
return { success: true, file_id }; return { success: true, file_id };
} catch (error) { } catch (error) {
@ -150,8 +157,9 @@ export const processAndUploadMedia = async (asset: ExtendedAsset) => {
} }
} }
// 5. 标记为已上传 // 标记为已上传
await markFileAsUploaded(asset.uri); await updateUploadTaskStatus(asset.uri, 'success', originalResult.file_id);
await updateUploadTaskProgress(asset.uri, 100);
return { return {
id: asset.id, id: asset.id,
@ -165,6 +173,9 @@ export const processAndUploadMedia = async (asset: ExtendedAsset) => {
} catch (error: any) { } catch (error: any) {
console.error('Error processing media upload for asset:', asset.uri, error); console.error('Error processing media upload for asset:', asset.uri, error);
// 标记为失败
await updateUploadTaskStatus(asset.uri, 'failed');
await updateUploadTaskProgress(asset.uri, 0);
return { return {
id: asset.id, id: asset.id,
originalSuccess: false, originalSuccess: false,

View File

@ -2,29 +2,94 @@ import * as SQLite from 'expo-sqlite';
const db = SQLite.openDatabaseSync('upload_status.db'); const db = SQLite.openDatabaseSync('upload_status.db');
export type UploadTask = {
uri: string;
filename: string;
status: 'pending' | 'uploading' | 'success' | 'failed' | 'skipped';
progress: number; // 0-100
file_id?: string; // 后端返回的文件ID
};
// 初始化表 // 初始化表
export function initUploadTable() { export function initUploadTable() {
console.log('Initializing upload table...'); console.log('Initializing upload tasks table...');
db.execSync(` db.execSync(`
CREATE TABLE IF NOT EXISTS uploaded_files ( CREATE TABLE IF NOT EXISTS upload_tasks (
uri TEXT PRIMARY KEY NOT NULL uri TEXT PRIMARY KEY NOT NULL,
filename TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'pending',
progress INTEGER NOT NULL DEFAULT 0,
file_id TEXT
); );
`); `);
console.log('Upload table initialized'); console.log('Upload tasks table initialized');
} }
// 检查文件是否已上传 (使用同步API但保持接口为Promise以减少外部重构) // 插入新的上传任务
export async function isFileUploaded(uri: string): Promise<boolean> { export async function insertUploadTask(uri: string, filename: string): Promise<void> {
console.log('Checking if file is uploaded:', uri) console.log('Inserting upload task:', uri, filename);
const result = db.getFirstSync<{ uri: string }>( db.runSync(
'SELECT uri FROM uploaded_files WHERE uri = ?;', 'INSERT OR IGNORE INTO upload_tasks (uri, filename, status, progress) VALUES (?, ?, ?, ?);',
uri,
filename,
'pending',
0
);
}
// 检查文件是否已上传或正在上传
export async function getUploadTaskStatus(uri: string): Promise<UploadTask | null> {
console.log('Checking upload task status for:', uri);
const result = db.getFirstSync<UploadTask>(
'SELECT uri, filename, status, progress, file_id FROM upload_tasks WHERE uri = ?;',
uri uri
); );
console.log('File uploaded result:', result) return result || null;
return !!result;
} }
// 记录文件已上传 // 更新上传任务的状态
export function markFileAsUploaded(uri: string) { export async function updateUploadTaskStatus(uri: string, status: UploadTask['status'], file_id?: string): Promise<void> {
db.runSync('INSERT OR IGNORE INTO uploaded_files (uri) VALUES (?);', uri); console.log('Updating upload task status:', uri, status, file_id);
if (file_id) {
db.runSync(
'UPDATE upload_tasks SET status = ?, file_id = ? WHERE uri = ?;',
status,
file_id,
uri
);
} else {
db.runSync(
'UPDATE upload_tasks SET status = ? WHERE uri = ?;',
status,
uri
);
}
} }
// 更新上传任务的进度
export async function updateUploadTaskProgress(uri: string, progress: number): Promise<void> {
console.log('Updating upload task progress:', uri, progress);
db.runSync(
'UPDATE upload_tasks SET progress = ? WHERE uri = ?;',
progress,
uri
);
}
// 获取所有上传任务
export async function getUploadTasks(): Promise<UploadTask[]> {
console.log('Fetching all upload tasks...');
const results = db.getAllSync<UploadTask>(
'SELECT uri, filename, status, progress, file_id FROM upload_tasks;'
);
return results;
}
// 清理已完成或失败的任务 (可选,根据需求添加)
export async function cleanUpUploadTasks(): Promise<void> {
console.log('Cleaning up completed/failed upload tasks...');
db.runSync(
"DELETE FROM upload_tasks WHERE status = 'success' OR status = 'failed' OR status = 'skipped';"
);
}