perf(backup,converter): optimize sql dump streaming, add direct server file conversion, and extend timeout to 30m

This commit is contained in:
Ray1111-aa
2026-11-03 10:11:58 +07:00
parent ff3ab616e0
commit 6e5c6eeeb8
7 changed files with 305 additions and 67 deletions
@@ -434,21 +434,36 @@ class BackupController extends Controller
public function converterInspect(Request $request): JsonResponse public function converterInspect(Request $request): JsonResponse
{ {
$request->validate([ $request->validate([
'backup_file' => 'required|file', 'backup_file' => 'nullable|file',
'server_filename' => 'nullable|string',
]); ]);
try {
if ($request->filled('server_filename')) {
$serverFile = $request->input('server_filename');
if (!str_ends_with(strtolower($serverFile), '.kmh')) {
return response()->json([
'success' => false,
'message' => 'Berkas yang dipilih harus berekstensi .kmh.',
], 422);
}
$metadata = $this->converterService->inspectServerFile($serverFile);
} elseif ($request->hasFile('backup_file')) {
$file = $request->file('backup_file'); $file = $request->file('backup_file');
$originalName = $file->getClientOriginalName(); $originalName = $file->getClientOriginalName();
if (!str_ends_with(strtolower($originalName), '.kmh')) { if (!str_ends_with(strtolower($originalName), '.kmh')) {
return response()->json([ return response()->json([
'success' => false, 'success' => false,
'message' => 'Berkas yang dipilih harus berekstensi .kmh.', 'message' => 'Berkas yang dipilih harus berekstensi .kmh.',
], 422); ], 422);
} }
try {
$metadata = $this->converterService->inspectUploadedFile($file); $metadata = $this->converterService->inspectUploadedFile($file);
} else {
return response()->json([
'success' => false,
'message' => 'Silakan pilih berkas .kmh dari komputer atau server.',
], 422);
}
return response()->json([ return response()->json([
'success' => true, 'success' => true,
+40 -28
View File
@@ -149,8 +149,8 @@ class DatabaseBackupService
*/ */
public function createKmhBackup(?array $selectedTables = null, bool $includeMedia = false, ?string $customFilename = null): string public function createKmhBackup(?array $selectedTables = null, bool $includeMedia = false, ?string $customFilename = null): string
{ {
@set_time_limit(600); @set_time_limit(1800);
@ini_set('memory_limit', '512M'); @ini_set('memory_limit', '1024M');
if (!class_exists('ZipArchive')) { if (!class_exists('ZipArchive')) {
throw new RuntimeException('Ekstensi PHP ZipArchive diperlukan untuk memaketkan arsip.'); throw new RuntimeException('Ekstensi PHP ZipArchive diperlukan untuk memaketkan arsip.');
@@ -242,8 +242,8 @@ class DatabaseBackupService
*/ */
public function restore(string $backupFilenameOrPath): void public function restore(string $backupFilenameOrPath): void
{ {
@set_time_limit(600); @set_time_limit(1800);
@ini_set('memory_limit', '512M'); @ini_set('memory_limit', '1024M');
$filePath = File::exists($backupFilenameOrPath) $filePath = File::exists($backupFilenameOrPath)
? $backupFilenameOrPath ? $backupFilenameOrPath
@@ -674,37 +674,37 @@ class DatabaseBackupService
fwrite($handle, "-- Dumping Data untuk `{$tableName}`\n"); fwrite($handle, "-- Dumping Data untuk `{$tableName}`\n");
$query = DB::table($tableName); $sql = "SELECT * FROM `{$tableName}`";
// Pastikan data yang berada di tong sampah (soft-deleted) tidak bocor ke dalam data backup
if (Schema::hasColumn($tableName, 'deleted_at')) { if (Schema::hasColumn($tableName, 'deleted_at')) {
$query->whereNull('deleted_at'); $sql .= " WHERE `deleted_at` IS NULL";
} }
$totalRows = $query->count(); try {
$stmt = $pdo->query($sql);
} catch (\Throwable $e) {
$stmt = null;
}
if ($totalRows === 0) { if (!$stmt) {
fwrite($handle, "-- (Tabel kosong)\n\n"); fwrite($handle, "-- (Tabel kosong atau gagal dibaca)\n\n");
return; return;
} }
$query->orderBy(DB::raw('1'))->chunk(200, function ($rows) use ($handle, $pdo, $tableName) { $batch = [];
if ($rows->isEmpty()) return; $batchSize = 500;
$columnsStr = null;
$hasData = false;
$columns = array_keys((array) $rows->first()); while ($row = $stmt->fetch(PDO::FETCH_ASSOC)) {
$hasData = true;
if ($columnsStr === null) {
$columns = array_keys($row);
$escapedColumns = array_map(fn ($c) => "`{$c}`", $columns); $escapedColumns = array_map(fn ($c) => "`{$c}`", $columns);
$columnsStr = implode(', ', $escapedColumns); $columnsStr = implode(', ', $escapedColumns);
}
fwrite($handle, "INSERT INTO `{$tableName}` ({$columnsStr}) VALUES\n");
$rowCount = count($rows);
$i = 0;
foreach ($rows as $row) {
$i++;
$values = []; $values = [];
foreach ($row as $val) {
foreach ((array) $row as $val) {
if (is_null($val)) { if (is_null($val)) {
$values[] = 'NULL'; $values[] = 'NULL';
} elseif (is_int($val) || is_float($val)) { } elseif (is_int($val) || is_float($val)) {
@@ -716,14 +716,26 @@ class DatabaseBackupService
} }
} }
$line = ' (' . implode(', ', $values) . ')'; $batch[] = ' (' . implode(', ', $values) . ')';
$line .= ($i === $rowCount) ? ";\n" : ",\n";
fwrite($handle, $line);
}
});
if (count($batch) >= $batchSize) {
fwrite($handle, "INSERT INTO `{$tableName}` ({$columnsStr}) VALUES\n");
fwrite($handle, implode(",\n", $batch) . ";\n");
$batch = [];
}
}
if (!empty($batch)) {
fwrite($handle, "INSERT INTO `{$tableName}` ({$columnsStr}) VALUES\n");
fwrite($handle, implode(",\n", $batch) . ";\n");
}
if (!$hasData) {
fwrite($handle, "-- (Tabel kosong)\n\n");
} else {
fwrite($handle, "\n"); fwrite($handle, "\n");
} }
}
/** /**
* Format byte ke satuan yang mudah dibaca (KB, MB, GB). * Format byte ke satuan yang mudah dibaca (KB, MB, GB).
+102
View File
@@ -14,6 +14,102 @@ class KmhConverterService
protected DatabaseBackupService $backupService protected DatabaseBackupService $backupService
) {} ) {}
/**
* Periksa metadata dan daftar tabel dari berkas .kmh yang sudah tersimpan di server.
* Tidak memerlukan pengunggahan ulang berkas dari PC (sangat cepat & ramah kuota).
*
* @return array{
* token: string,
* filename: string,
* size_human: string,
* created_at_human: string,
* created_at_diff: string,
* table_count: int,
* tables: array<string>,
* database: string,
* has_media: bool
* }
*/
public function inspectServerFile(string $filename): array
{
@set_time_limit(1800);
@ini_set('memory_limit', '1024M');
$backupDir = $this->backupService->getBackupDirectory();
$realPath = $backupDir . DIRECTORY_SEPARATOR . basename($filename);
if (!is_file($realPath)) {
throw new RuntimeException("Berkas arsip '{$filename}' tidak ditemukan di penyimpanan server.");
}
$kmhContent = file_get_contents($realPath);
if ($kmhContent === false) {
throw new RuntimeException("Gagal membaca berkas .kmh dari server.");
}
$decryptedZipContent = $this->backupService->decryptFromKmh($kmhContent);
unset($kmhContent);
$tempZipPath = storage_path('app/temp_inspect_' . uniqid() . '.zip');
file_put_contents($tempZipPath, $decryptedZipContent);
unset($decryptedZipContent);
$zip = new ZipArchive();
if ($zip->open($tempZipPath) !== true) {
@unlink($tempZipPath);
throw new RuntimeException("Konten terdekripsi bukan merupakan arsip ZIP yang sah.");
}
try {
$manifestJson = $zip->getFromName('manifest.json');
$manifest = $manifestJson ? json_decode($manifestJson, true) : [];
$tables = $manifest['tables'] ?? [];
if (empty($tables)) {
$sqlContent = $zip->getFromName('database.sql');
if ($sqlContent) {
preg_match_all('/CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?`?([a-zA-Z0-9_]+)`?/i', $sqlContent, $matches);
$tables = array_values(array_unique($matches[1] ?? []));
}
}
sort($tables);
$createdAt = isset($manifest['created_at'])
? Carbon::parse($manifest['created_at'])
: Carbon::createFromTimestamp(filemtime($realPath));
$tempDir = storage_path('app/temp_converter');
File::ensureDirectoryExists($tempDir);
// Bersihkan file sementara yang lebih tua dari 30 menit
foreach (File::files($tempDir) as $f) {
if ((time() - $f->getMTime()) > 1800) {
@unlink($f->getPathname());
}
}
$token = bin2hex(random_bytes(16));
$cachedKmhPath = $tempDir . '/' . $token . '.kmh';
copy($realPath, $cachedKmhPath);
return [
'token' => $token,
'filename' => basename($filename),
'size_human' => $this->backupService->formatBytes((int) filesize($realPath)),
'created_at_human' => $createdAt->translatedFormat('d F Y, H:i') . ' WIB',
'created_at_diff' => $createdAt->diffForHumans(),
'table_count' => count($tables),
'tables' => $tables,
'database' => $manifest['database'] ?? config('database.connections.' . config('database.default') . '.database', 'rkm_kmh'),
'has_media' => (bool) ($manifest['has_media'] ?? false),
];
} finally {
$zip->close();
@unlink($tempZipPath);
}
}
/** /**
* Periksa metadata dan daftar tabel dari berkas .kmh yang diunggah dari PC. * Periksa metadata dan daftar tabel dari berkas .kmh yang diunggah dari PC.
* Menyimpan berkas sementara dengan token aman agar saat klik konversi tidak perlu upload ulang 2x. * Menyimpan berkas sementara dengan token aman agar saat klik konversi tidak perlu upload ulang 2x.
@@ -32,6 +128,9 @@ class KmhConverterService
*/ */
public function inspectUploadedFile(UploadedFile $file): array public function inspectUploadedFile(UploadedFile $file): array
{ {
@set_time_limit(1800);
@ini_set('memory_limit', '1024M');
$realPath = $file->getRealPath(); $realPath = $file->getRealPath();
if (!$realPath || !is_file($realPath)) { if (!$realPath || !is_file($realPath)) {
throw new RuntimeException("Berkas unggahan tidak ditemukan atau korup."); throw new RuntimeException("Berkas unggahan tidak ditemukan atau korup.");
@@ -160,6 +259,9 @@ class KmhConverterService
*/ */
public function convertKmhFilePath(string $kmhFilePath, string $originalFilename, array $selectedTables, string $format): array public function convertKmhFilePath(string $kmhFilePath, string $originalFilename, array $selectedTables, string $format): array
{ {
@set_time_limit(1800);
@ini_set('memory_limit', '1024M');
$format = strtolower($format); $format = strtolower($format);
if (!in_array($format, ['xlsx', 'csv', 'sql', 'media'], true)) { if (!in_array($format, ['xlsx', 'csv', 'sql', 'media'], true)) {
throw new RuntimeException("Format konversi tidak didukung: {$format}"); throw new RuntimeException("Format konversi tidak didukung: {$format}");
+6 -6
View File
@@ -10,10 +10,10 @@ server {
client_max_body_size 500M; client_max_body_size 500M;
client_body_buffer_size 1M; client_body_buffer_size 1M;
client_body_timeout 600s; client_body_timeout 1800s;
client_header_timeout 600s; client_header_timeout 1800s;
send_timeout 600s; send_timeout 1800s;
keepalive_timeout 600s; keepalive_timeout 1800s;
# Security Headers # Security Headers
add_header X-Frame-Options "SAMEORIGIN" always; add_header X-Frame-Options "SAMEORIGIN" always;
@@ -61,8 +61,8 @@ server {
fastcgi_hide_header X-Powered-By; fastcgi_hide_header X-Powered-By;
fastcgi_buffers 16 16k; fastcgi_buffers 16 16k;
fastcgi_buffer_size 32k; fastcgi_buffer_size 32k;
fastcgi_read_timeout 600; fastcgi_read_timeout 1800;
fastcgi_send_timeout 600; fastcgi_send_timeout 1800;
} }
location ~ /\.(?!well-known).* { location ~ /\.(?!well-known).* {
+2 -2
View File
@@ -1,7 +1,7 @@
upload_max_filesize = 500M upload_max_filesize = 500M
post_max_size = 500M post_max_size = 500M
memory_limit = 1024M memory_limit = 1024M
max_execution_time = 600 max_execution_time = 1800
max_input_time = 600 max_input_time = 1800
date.timezone = Asia/Jakarta date.timezone = Asia/Jakarta
expose_php = Off expose_php = Off
+84 -4
View File
@@ -235,6 +235,13 @@
Unduh Unduh
</a> </a>
<button type="button"
onclick="openServerConverterModal('{{ $log->filename }}')"
title="Konversi langsung berkas arsip ini ke Excel / CSV / SQL tanpa perlu upload ulang dari PC"
class="px-2.5 py-1 rounded-lg border border-indigo-200 bg-indigo-50 hover:bg-indigo-100 text-indigo-800 font-semibold text-[11px] transition">
Konversi
</button>
<button type="button" <button type="button"
onclick="openRestoreModal('{{ $log->filename }}', '{{ $log->scope_label }}', '{{ $log->file_size_human }}')" onclick="openRestoreModal('{{ $log->filename }}', '{{ $log->scope_label }}', '{{ $log->file_size_human }}')"
class="px-2.5 py-1 rounded-lg border border-amber-300 bg-amber-50 hover:bg-amber-100 text-amber-900 font-semibold text-[11px] transition"> class="px-2.5 py-1 rounded-lg border border-amber-300 bg-amber-50 hover:bg-amber-100 text-amber-900 font-semibold text-[11px] transition">
@@ -1012,7 +1019,7 @@
const xhr = new XMLHttpRequest(); const xhr = new XMLHttpRequest();
xhr.open('POST', form.action, true); xhr.open('POST', form.action, true);
xhr.timeout = 600000; // 10 Menit timeout untuk berkas backup besar xhr.timeout = 1800000; // 30 Menit timeout untuk berkas backup besar
xhr.setRequestHeader('X-Requested-With', 'XMLHttpRequest'); xhr.setRequestHeader('X-Requested-With', 'XMLHttpRequest');
xhr.upload.onprogress = function(e) { xhr.upload.onprogress = function(e) {
@@ -1083,7 +1090,7 @@
btn.classList.remove('opacity-75', 'cursor-not-allowed'); btn.classList.remove('opacity-75', 'cursor-not-allowed');
} }
if (text) text.textContent = 'Mulai Pemulihan Sistem'; if (text) text.textContent = 'Mulai Pemulihan Sistem';
document.getElementById('directRestoreErrorMessage').textContent = 'Waktu pengunggahan habis (Timeout 10 Menit). Silakan periksa kestabilan jaringan internet Anda.'; document.getElementById('directRestoreErrorMessage').textContent = 'Waktu pengunggahan habis (Timeout 30 Menit). Silakan periksa kestabilan jaringan internet Anda.';
document.getElementById('directRestoreError').classList.remove('hidden'); document.getElementById('directRestoreError').classList.remove('hidden');
}; };
@@ -1222,6 +1229,79 @@
} }
} }
function openServerConverterModal(filename) {
openConverterModal();
// Sembunyikan panel upload dan tampilkan status loading membaca arsip server
document.getElementById('converterError').classList.add('hidden');
document.getElementById('panelConverterUpload').classList.add('hidden');
document.getElementById('panelConverterConfig').classList.add('hidden');
const loading = document.getElementById('converterLoading');
const progressBar = document.getElementById('converterProgressBar');
const percentText = document.getElementById('converterUploadPercentText');
const byteText = document.getElementById('converterUploadByteText');
const statusText = document.getElementById('converterLoadingText');
const subText = document.getElementById('converterUploadSubText');
progressBar.style.width = '100%';
percentText.textContent = '100%';
byteText.textContent = 'Arsip Server';
statusText.textContent = `Membaca arsip server "${filename}"...`;
subText.textContent = 'Membaca manifest & struktur tabel langsung dari VPS (Tanpa upload)...';
loading.classList.remove('hidden');
const formData = new FormData();
formData.append('server_filename', filename);
formData.append('_token', '{{ csrf_token() }}');
fetch(`{{ route('admin.backups.converter.inspect') }}`, {
method: 'POST',
body: formData,
headers: {
'X-Requested-With': 'XMLHttpRequest',
'Accept': 'application/json'
}
})
.then(async response => {
loading.classList.add('hidden');
const res = await response.json().catch(() => ({}));
if (response.ok && res.success && res.data) {
currentConverterFile = null;
currentConverterToken = res.data.token || null;
currentConverterTables = res.data.tables || [];
document.getElementById('convFilename').textContent = res.data.filename;
document.getElementById('convFileSize').textContent = res.data.size_human;
document.getElementById('convCreatedAt').textContent = res.data.created_at_human;
document.getElementById('convDatabase').textContent = res.data.database;
const mediaCard = document.getElementById('convMediaFormatCard');
if (mediaCard) {
if (res.data.has_media) {
mediaCard.classList.remove('hidden');
} else {
mediaCard.classList.add('hidden');
}
}
const filterInput = document.getElementById('convTableFilterInput');
if (filterInput) filterInput.value = '';
renderConverterTableChecklist('');
document.getElementById('panelConverterConfig').classList.remove('hidden');
} else {
document.getElementById('panelConverterUpload').classList.remove('hidden');
showConverterError(res.message || 'Gagal membaca berkas arsip .kmh dari server.');
}
})
.catch(err => {
loading.classList.add('hidden');
document.getElementById('panelConverterUpload').classList.remove('hidden');
showConverterError(err.message || 'Terjadi kesalahan saat memeriksa berkas server.');
});
}
function inspectConverterFile(file) { function inspectConverterFile(file) {
if (!file.name.toLowerCase().endsWith('.kmh')) { if (!file.name.toLowerCase().endsWith('.kmh')) {
alert('File yang dipilih harus berekstensi .kmh!'); alert('File yang dipilih harus berekstensi .kmh!');
@@ -1252,7 +1332,7 @@
const xhr = new XMLHttpRequest(); const xhr = new XMLHttpRequest();
xhr.open('POST', `{{ route('admin.backups.converter.inspect') }}`, true); xhr.open('POST', `{{ route('admin.backups.converter.inspect') }}`, true);
xhr.timeout = 600000; // 10 Menit timeout xhr.timeout = 1800000; // 30 Menit timeout
xhr.setRequestHeader('X-Requested-With', 'XMLHttpRequest'); xhr.setRequestHeader('X-Requested-With', 'XMLHttpRequest');
xhr.setRequestHeader('Accept', 'application/json'); xhr.setRequestHeader('Accept', 'application/json');
@@ -1331,7 +1411,7 @@
xhr.ontimeout = function() { xhr.ontimeout = function() {
loading.classList.add('hidden'); loading.classList.add('hidden');
document.getElementById('panelConverterUpload').classList.remove('hidden'); document.getElementById('panelConverterUpload').classList.remove('hidden');
showConverterError('Waktu pengunggahan habis (Timeout 10 Menit). Silakan periksa kestabilan jaringan internet Anda.'); showConverterError('Waktu pengunggahan habis (Timeout 30 Menit). Silakan periksa kestabilan jaringan internet Anda.');
}; };
xhr.send(formData); xhr.send(formData);
+29
View File
@@ -310,4 +310,33 @@ class BackupAndRestoreTest extends TestCase
'target_name' => 'backup_dari_pc.kmh', 'target_name' => 'backup_dari_pc.kmh',
]); ]);
} }
public function test_super_admin_can_inspect_and_convert_server_backup_file(): void
{
$filename = $this->backupService->createKmhBackup(['users', 'roles'], false, 'test_server_conv.kmh');
// 1. Inspect server file via AJAX
$inspectResponse = $this->actingAs($this->superAdmin)
->postJson(route('admin.backups.converter.inspect'), [
'server_filename' => $filename,
]);
$inspectResponse->assertStatus(200);
$inspectResponse->assertJson([
'success' => true,
]);
$token = $inspectResponse->json('data.token');
$this->assertNotEmpty($token);
// 2. Convert using token to SQL
$convResponse = $this->actingAs($this->superAdmin)
->post(route('admin.backups.converter.execute'), [
'token' => $token,
'tables' => ['users'],
'format' => 'sql',
]);
$convResponse->assertStatus(200);
$this->assertEquals('application/sql', $convResponse->headers->get('Content-Type'));
}
} }