perf(backup,converter): optimize sql dump streaming, add direct server file conversion, and extend timeout to 30m
This commit is contained in:
@@ -434,21 +434,36 @@ class BackupController extends Controller
|
||||
public function converterInspect(Request $request): JsonResponse
|
||||
{
|
||||
$request->validate([
|
||||
'backup_file' => 'required|file',
|
||||
'backup_file' => 'nullable|file',
|
||||
'server_filename' => 'nullable|string',
|
||||
]);
|
||||
|
||||
$file = $request->file('backup_file');
|
||||
$originalName = $file->getClientOriginalName();
|
||||
|
||||
if (!str_ends_with(strtolower($originalName), '.kmh')) {
|
||||
return response()->json([
|
||||
'success' => false,
|
||||
'message' => 'Berkas yang dipilih harus berekstensi .kmh.',
|
||||
], 422);
|
||||
}
|
||||
|
||||
try {
|
||||
$metadata = $this->converterService->inspectUploadedFile($file);
|
||||
if ($request->filled('server_filename')) {
|
||||
$serverFile = $request->input('server_filename');
|
||||
if (!str_ends_with(strtolower($serverFile), '.kmh')) {
|
||||
return response()->json([
|
||||
'success' => false,
|
||||
'message' => 'Berkas yang dipilih harus berekstensi .kmh.',
|
||||
], 422);
|
||||
}
|
||||
$metadata = $this->converterService->inspectServerFile($serverFile);
|
||||
} elseif ($request->hasFile('backup_file')) {
|
||||
$file = $request->file('backup_file');
|
||||
$originalName = $file->getClientOriginalName();
|
||||
if (!str_ends_with(strtolower($originalName), '.kmh')) {
|
||||
return response()->json([
|
||||
'success' => false,
|
||||
'message' => 'Berkas yang dipilih harus berekstensi .kmh.',
|
||||
], 422);
|
||||
}
|
||||
$metadata = $this->converterService->inspectUploadedFile($file);
|
||||
} else {
|
||||
return response()->json([
|
||||
'success' => false,
|
||||
'message' => 'Silakan pilih berkas .kmh dari komputer atau server.',
|
||||
], 422);
|
||||
}
|
||||
|
||||
return response()->json([
|
||||
'success' => true,
|
||||
|
||||
@@ -149,8 +149,8 @@ class DatabaseBackupService
|
||||
*/
|
||||
public function createKmhBackup(?array $selectedTables = null, bool $includeMedia = false, ?string $customFilename = null): string
|
||||
{
|
||||
@set_time_limit(600);
|
||||
@ini_set('memory_limit', '512M');
|
||||
@set_time_limit(1800);
|
||||
@ini_set('memory_limit', '1024M');
|
||||
|
||||
if (!class_exists('ZipArchive')) {
|
||||
throw new RuntimeException('Ekstensi PHP ZipArchive diperlukan untuk memaketkan arsip.');
|
||||
@@ -242,8 +242,8 @@ class DatabaseBackupService
|
||||
*/
|
||||
public function restore(string $backupFilenameOrPath): void
|
||||
{
|
||||
@set_time_limit(600);
|
||||
@ini_set('memory_limit', '512M');
|
||||
@set_time_limit(1800);
|
||||
@ini_set('memory_limit', '1024M');
|
||||
|
||||
$filePath = File::exists($backupFilenameOrPath)
|
||||
? $backupFilenameOrPath
|
||||
@@ -674,55 +674,67 @@ class DatabaseBackupService
|
||||
|
||||
fwrite($handle, "-- Dumping Data untuk `{$tableName}`\n");
|
||||
|
||||
$query = DB::table($tableName);
|
||||
|
||||
// Pastikan data yang berada di tong sampah (soft-deleted) tidak bocor ke dalam data backup
|
||||
$sql = "SELECT * FROM `{$tableName}`";
|
||||
if (Schema::hasColumn($tableName, 'deleted_at')) {
|
||||
$query->whereNull('deleted_at');
|
||||
$sql .= " WHERE `deleted_at` IS NULL";
|
||||
}
|
||||
|
||||
$totalRows = $query->count();
|
||||
try {
|
||||
$stmt = $pdo->query($sql);
|
||||
} catch (\Throwable $e) {
|
||||
$stmt = null;
|
||||
}
|
||||
|
||||
if ($totalRows === 0) {
|
||||
fwrite($handle, "-- (Tabel kosong)\n\n");
|
||||
if (!$stmt) {
|
||||
fwrite($handle, "-- (Tabel kosong atau gagal dibaca)\n\n");
|
||||
return;
|
||||
}
|
||||
|
||||
$query->orderBy(DB::raw('1'))->chunk(200, function ($rows) use ($handle, $pdo, $tableName) {
|
||||
if ($rows->isEmpty()) return;
|
||||
$batch = [];
|
||||
$batchSize = 500;
|
||||
$columnsStr = null;
|
||||
$hasData = false;
|
||||
|
||||
$columns = array_keys((array) $rows->first());
|
||||
$escapedColumns = array_map(fn ($c) => "`{$c}`", $columns);
|
||||
$columnsStr = implode(', ', $escapedColumns);
|
||||
|
||||
fwrite($handle, "INSERT INTO `{$tableName}` ({$columnsStr}) VALUES\n");
|
||||
|
||||
$rowCount = count($rows);
|
||||
$i = 0;
|
||||
|
||||
foreach ($rows as $row) {
|
||||
$i++;
|
||||
$values = [];
|
||||
|
||||
foreach ((array) $row as $val) {
|
||||
if (is_null($val)) {
|
||||
$values[] = 'NULL';
|
||||
} elseif (is_int($val) || is_float($val)) {
|
||||
$values[] = (string) $val;
|
||||
} elseif (is_bool($val)) {
|
||||
$values[] = $val ? '1' : '0';
|
||||
} else {
|
||||
$values[] = $pdo->quote((string) $val);
|
||||
}
|
||||
}
|
||||
|
||||
$line = ' (' . implode(', ', $values) . ')';
|
||||
$line .= ($i === $rowCount) ? ";\n" : ",\n";
|
||||
fwrite($handle, $line);
|
||||
while ($row = $stmt->fetch(PDO::FETCH_ASSOC)) {
|
||||
$hasData = true;
|
||||
if ($columnsStr === null) {
|
||||
$columns = array_keys($row);
|
||||
$escapedColumns = array_map(fn ($c) => "`{$c}`", $columns);
|
||||
$columnsStr = implode(', ', $escapedColumns);
|
||||
}
|
||||
});
|
||||
|
||||
fwrite($handle, "\n");
|
||||
$values = [];
|
||||
foreach ($row as $val) {
|
||||
if (is_null($val)) {
|
||||
$values[] = 'NULL';
|
||||
} elseif (is_int($val) || is_float($val)) {
|
||||
$values[] = (string) $val;
|
||||
} elseif (is_bool($val)) {
|
||||
$values[] = $val ? '1' : '0';
|
||||
} else {
|
||||
$values[] = $pdo->quote((string) $val);
|
||||
}
|
||||
}
|
||||
|
||||
$batch[] = ' (' . implode(', ', $values) . ')';
|
||||
|
||||
if (count($batch) >= $batchSize) {
|
||||
fwrite($handle, "INSERT INTO `{$tableName}` ({$columnsStr}) VALUES\n");
|
||||
fwrite($handle, implode(",\n", $batch) . ";\n");
|
||||
$batch = [];
|
||||
}
|
||||
}
|
||||
|
||||
if (!empty($batch)) {
|
||||
fwrite($handle, "INSERT INTO `{$tableName}` ({$columnsStr}) VALUES\n");
|
||||
fwrite($handle, implode(",\n", $batch) . ";\n");
|
||||
}
|
||||
|
||||
if (!$hasData) {
|
||||
fwrite($handle, "-- (Tabel kosong)\n\n");
|
||||
} else {
|
||||
fwrite($handle, "\n");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -14,6 +14,102 @@ class KmhConverterService
|
||||
protected DatabaseBackupService $backupService
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Periksa metadata dan daftar tabel dari berkas .kmh yang sudah tersimpan di server.
|
||||
* Tidak memerlukan pengunggahan ulang berkas dari PC (sangat cepat & ramah kuota).
|
||||
*
|
||||
* @return array{
|
||||
* token: string,
|
||||
* filename: string,
|
||||
* size_human: string,
|
||||
* created_at_human: string,
|
||||
* created_at_diff: string,
|
||||
* table_count: int,
|
||||
* tables: array<string>,
|
||||
* database: string,
|
||||
* has_media: bool
|
||||
* }
|
||||
*/
|
||||
public function inspectServerFile(string $filename): array
|
||||
{
|
||||
@set_time_limit(1800);
|
||||
@ini_set('memory_limit', '1024M');
|
||||
|
||||
$backupDir = $this->backupService->getBackupDirectory();
|
||||
$realPath = $backupDir . DIRECTORY_SEPARATOR . basename($filename);
|
||||
if (!is_file($realPath)) {
|
||||
throw new RuntimeException("Berkas arsip '{$filename}' tidak ditemukan di penyimpanan server.");
|
||||
}
|
||||
|
||||
$kmhContent = file_get_contents($realPath);
|
||||
if ($kmhContent === false) {
|
||||
throw new RuntimeException("Gagal membaca berkas .kmh dari server.");
|
||||
}
|
||||
|
||||
$decryptedZipContent = $this->backupService->decryptFromKmh($kmhContent);
|
||||
unset($kmhContent);
|
||||
|
||||
$tempZipPath = storage_path('app/temp_inspect_' . uniqid() . '.zip');
|
||||
file_put_contents($tempZipPath, $decryptedZipContent);
|
||||
unset($decryptedZipContent);
|
||||
|
||||
$zip = new ZipArchive();
|
||||
if ($zip->open($tempZipPath) !== true) {
|
||||
@unlink($tempZipPath);
|
||||
throw new RuntimeException("Konten terdekripsi bukan merupakan arsip ZIP yang sah.");
|
||||
}
|
||||
|
||||
try {
|
||||
$manifestJson = $zip->getFromName('manifest.json');
|
||||
$manifest = $manifestJson ? json_decode($manifestJson, true) : [];
|
||||
|
||||
$tables = $manifest['tables'] ?? [];
|
||||
|
||||
if (empty($tables)) {
|
||||
$sqlContent = $zip->getFromName('database.sql');
|
||||
if ($sqlContent) {
|
||||
preg_match_all('/CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?`?([a-zA-Z0-9_]+)`?/i', $sqlContent, $matches);
|
||||
$tables = array_values(array_unique($matches[1] ?? []));
|
||||
}
|
||||
}
|
||||
|
||||
sort($tables);
|
||||
|
||||
$createdAt = isset($manifest['created_at'])
|
||||
? Carbon::parse($manifest['created_at'])
|
||||
: Carbon::createFromTimestamp(filemtime($realPath));
|
||||
|
||||
$tempDir = storage_path('app/temp_converter');
|
||||
File::ensureDirectoryExists($tempDir);
|
||||
|
||||
// Bersihkan file sementara yang lebih tua dari 30 menit
|
||||
foreach (File::files($tempDir) as $f) {
|
||||
if ((time() - $f->getMTime()) > 1800) {
|
||||
@unlink($f->getPathname());
|
||||
}
|
||||
}
|
||||
|
||||
$token = bin2hex(random_bytes(16));
|
||||
$cachedKmhPath = $tempDir . '/' . $token . '.kmh';
|
||||
copy($realPath, $cachedKmhPath);
|
||||
|
||||
return [
|
||||
'token' => $token,
|
||||
'filename' => basename($filename),
|
||||
'size_human' => $this->backupService->formatBytes((int) filesize($realPath)),
|
||||
'created_at_human' => $createdAt->translatedFormat('d F Y, H:i') . ' WIB',
|
||||
'created_at_diff' => $createdAt->diffForHumans(),
|
||||
'table_count' => count($tables),
|
||||
'tables' => $tables,
|
||||
'database' => $manifest['database'] ?? config('database.connections.' . config('database.default') . '.database', 'rkm_kmh'),
|
||||
'has_media' => (bool) ($manifest['has_media'] ?? false),
|
||||
];
|
||||
} finally {
|
||||
$zip->close();
|
||||
@unlink($tempZipPath);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Periksa metadata dan daftar tabel dari berkas .kmh yang diunggah dari PC.
|
||||
* Menyimpan berkas sementara dengan token aman agar saat klik konversi tidak perlu upload ulang 2x.
|
||||
@@ -32,6 +128,9 @@ class KmhConverterService
|
||||
*/
|
||||
public function inspectUploadedFile(UploadedFile $file): array
|
||||
{
|
||||
@set_time_limit(1800);
|
||||
@ini_set('memory_limit', '1024M');
|
||||
|
||||
$realPath = $file->getRealPath();
|
||||
if (!$realPath || !is_file($realPath)) {
|
||||
throw new RuntimeException("Berkas unggahan tidak ditemukan atau korup.");
|
||||
@@ -160,6 +259,9 @@ class KmhConverterService
|
||||
*/
|
||||
public function convertKmhFilePath(string $kmhFilePath, string $originalFilename, array $selectedTables, string $format): array
|
||||
{
|
||||
@set_time_limit(1800);
|
||||
@ini_set('memory_limit', '1024M');
|
||||
|
||||
$format = strtolower($format);
|
||||
if (!in_array($format, ['xlsx', 'csv', 'sql', 'media'], true)) {
|
||||
throw new RuntimeException("Format konversi tidak didukung: {$format}");
|
||||
|
||||
@@ -10,10 +10,10 @@ server {
|
||||
|
||||
client_max_body_size 500M;
|
||||
client_body_buffer_size 1M;
|
||||
client_body_timeout 600s;
|
||||
client_header_timeout 600s;
|
||||
send_timeout 600s;
|
||||
keepalive_timeout 600s;
|
||||
client_body_timeout 1800s;
|
||||
client_header_timeout 1800s;
|
||||
send_timeout 1800s;
|
||||
keepalive_timeout 1800s;
|
||||
|
||||
# Security Headers
|
||||
add_header X-Frame-Options "SAMEORIGIN" always;
|
||||
@@ -61,8 +61,8 @@ server {
|
||||
fastcgi_hide_header X-Powered-By;
|
||||
fastcgi_buffers 16 16k;
|
||||
fastcgi_buffer_size 32k;
|
||||
fastcgi_read_timeout 600;
|
||||
fastcgi_send_timeout 600;
|
||||
fastcgi_read_timeout 1800;
|
||||
fastcgi_send_timeout 1800;
|
||||
}
|
||||
|
||||
location ~ /\.(?!well-known).* {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
upload_max_filesize = 500M
|
||||
post_max_size = 500M
|
||||
memory_limit = 1024M
|
||||
max_execution_time = 600
|
||||
max_input_time = 600
|
||||
max_execution_time = 1800
|
||||
max_input_time = 1800
|
||||
date.timezone = Asia/Jakarta
|
||||
expose_php = Off
|
||||
|
||||
@@ -235,6 +235,13 @@
|
||||
Unduh
|
||||
</a>
|
||||
|
||||
<button type="button"
|
||||
onclick="openServerConverterModal('{{ $log->filename }}')"
|
||||
title="Konversi langsung berkas arsip ini ke Excel / CSV / SQL tanpa perlu upload ulang dari PC"
|
||||
class="px-2.5 py-1 rounded-lg border border-indigo-200 bg-indigo-50 hover:bg-indigo-100 text-indigo-800 font-semibold text-[11px] transition">
|
||||
Konversi
|
||||
</button>
|
||||
|
||||
<button type="button"
|
||||
onclick="openRestoreModal('{{ $log->filename }}', '{{ $log->scope_label }}', '{{ $log->file_size_human }}')"
|
||||
class="px-2.5 py-1 rounded-lg border border-amber-300 bg-amber-50 hover:bg-amber-100 text-amber-900 font-semibold text-[11px] transition">
|
||||
@@ -1012,7 +1019,7 @@
|
||||
|
||||
const xhr = new XMLHttpRequest();
|
||||
xhr.open('POST', form.action, true);
|
||||
xhr.timeout = 600000; // 10 Menit timeout untuk berkas backup besar
|
||||
xhr.timeout = 1800000; // 30 Menit timeout untuk berkas backup besar
|
||||
xhr.setRequestHeader('X-Requested-With', 'XMLHttpRequest');
|
||||
|
||||
xhr.upload.onprogress = function(e) {
|
||||
@@ -1083,7 +1090,7 @@
|
||||
btn.classList.remove('opacity-75', 'cursor-not-allowed');
|
||||
}
|
||||
if (text) text.textContent = 'Mulai Pemulihan Sistem';
|
||||
document.getElementById('directRestoreErrorMessage').textContent = 'Waktu pengunggahan habis (Timeout 10 Menit). Silakan periksa kestabilan jaringan internet Anda.';
|
||||
document.getElementById('directRestoreErrorMessage').textContent = 'Waktu pengunggahan habis (Timeout 30 Menit). Silakan periksa kestabilan jaringan internet Anda.';
|
||||
document.getElementById('directRestoreError').classList.remove('hidden');
|
||||
};
|
||||
|
||||
@@ -1222,6 +1229,79 @@
|
||||
}
|
||||
}
|
||||
|
||||
function openServerConverterModal(filename) {
|
||||
openConverterModal();
|
||||
|
||||
// Sembunyikan panel upload dan tampilkan status loading membaca arsip server
|
||||
document.getElementById('converterError').classList.add('hidden');
|
||||
document.getElementById('panelConverterUpload').classList.add('hidden');
|
||||
document.getElementById('panelConverterConfig').classList.add('hidden');
|
||||
|
||||
const loading = document.getElementById('converterLoading');
|
||||
const progressBar = document.getElementById('converterProgressBar');
|
||||
const percentText = document.getElementById('converterUploadPercentText');
|
||||
const byteText = document.getElementById('converterUploadByteText');
|
||||
const statusText = document.getElementById('converterLoadingText');
|
||||
const subText = document.getElementById('converterUploadSubText');
|
||||
|
||||
progressBar.style.width = '100%';
|
||||
percentText.textContent = '100%';
|
||||
byteText.textContent = 'Arsip Server';
|
||||
statusText.textContent = `Membaca arsip server "${filename}"...`;
|
||||
subText.textContent = 'Membaca manifest & struktur tabel langsung dari VPS (Tanpa upload)...';
|
||||
loading.classList.remove('hidden');
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append('server_filename', filename);
|
||||
formData.append('_token', '{{ csrf_token() }}');
|
||||
|
||||
fetch(`{{ route('admin.backups.converter.inspect') }}`, {
|
||||
method: 'POST',
|
||||
body: formData,
|
||||
headers: {
|
||||
'X-Requested-With': 'XMLHttpRequest',
|
||||
'Accept': 'application/json'
|
||||
}
|
||||
})
|
||||
.then(async response => {
|
||||
loading.classList.add('hidden');
|
||||
const res = await response.json().catch(() => ({}));
|
||||
if (response.ok && res.success && res.data) {
|
||||
currentConverterFile = null;
|
||||
currentConverterToken = res.data.token || null;
|
||||
currentConverterTables = res.data.tables || [];
|
||||
|
||||
document.getElementById('convFilename').textContent = res.data.filename;
|
||||
document.getElementById('convFileSize').textContent = res.data.size_human;
|
||||
document.getElementById('convCreatedAt').textContent = res.data.created_at_human;
|
||||
document.getElementById('convDatabase').textContent = res.data.database;
|
||||
|
||||
const mediaCard = document.getElementById('convMediaFormatCard');
|
||||
if (mediaCard) {
|
||||
if (res.data.has_media) {
|
||||
mediaCard.classList.remove('hidden');
|
||||
} else {
|
||||
mediaCard.classList.add('hidden');
|
||||
}
|
||||
}
|
||||
|
||||
const filterInput = document.getElementById('convTableFilterInput');
|
||||
if (filterInput) filterInput.value = '';
|
||||
|
||||
renderConverterTableChecklist('');
|
||||
document.getElementById('panelConverterConfig').classList.remove('hidden');
|
||||
} else {
|
||||
document.getElementById('panelConverterUpload').classList.remove('hidden');
|
||||
showConverterError(res.message || 'Gagal membaca berkas arsip .kmh dari server.');
|
||||
}
|
||||
})
|
||||
.catch(err => {
|
||||
loading.classList.add('hidden');
|
||||
document.getElementById('panelConverterUpload').classList.remove('hidden');
|
||||
showConverterError(err.message || 'Terjadi kesalahan saat memeriksa berkas server.');
|
||||
});
|
||||
}
|
||||
|
||||
function inspectConverterFile(file) {
|
||||
if (!file.name.toLowerCase().endsWith('.kmh')) {
|
||||
alert('File yang dipilih harus berekstensi .kmh!');
|
||||
@@ -1252,7 +1332,7 @@
|
||||
|
||||
const xhr = new XMLHttpRequest();
|
||||
xhr.open('POST', `{{ route('admin.backups.converter.inspect') }}`, true);
|
||||
xhr.timeout = 600000; // 10 Menit timeout
|
||||
xhr.timeout = 1800000; // 30 Menit timeout
|
||||
xhr.setRequestHeader('X-Requested-With', 'XMLHttpRequest');
|
||||
xhr.setRequestHeader('Accept', 'application/json');
|
||||
|
||||
@@ -1331,7 +1411,7 @@
|
||||
xhr.ontimeout = function() {
|
||||
loading.classList.add('hidden');
|
||||
document.getElementById('panelConverterUpload').classList.remove('hidden');
|
||||
showConverterError('Waktu pengunggahan habis (Timeout 10 Menit). Silakan periksa kestabilan jaringan internet Anda.');
|
||||
showConverterError('Waktu pengunggahan habis (Timeout 30 Menit). Silakan periksa kestabilan jaringan internet Anda.');
|
||||
};
|
||||
|
||||
xhr.send(formData);
|
||||
|
||||
@@ -310,4 +310,33 @@ class BackupAndRestoreTest extends TestCase
|
||||
'target_name' => 'backup_dari_pc.kmh',
|
||||
]);
|
||||
}
|
||||
|
||||
public function test_super_admin_can_inspect_and_convert_server_backup_file(): void
|
||||
{
|
||||
$filename = $this->backupService->createKmhBackup(['users', 'roles'], false, 'test_server_conv.kmh');
|
||||
|
||||
// 1. Inspect server file via AJAX
|
||||
$inspectResponse = $this->actingAs($this->superAdmin)
|
||||
->postJson(route('admin.backups.converter.inspect'), [
|
||||
'server_filename' => $filename,
|
||||
]);
|
||||
|
||||
$inspectResponse->assertStatus(200);
|
||||
$inspectResponse->assertJson([
|
||||
'success' => true,
|
||||
]);
|
||||
$token = $inspectResponse->json('data.token');
|
||||
$this->assertNotEmpty($token);
|
||||
|
||||
// 2. Convert using token to SQL
|
||||
$convResponse = $this->actingAs($this->superAdmin)
|
||||
->post(route('admin.backups.converter.execute'), [
|
||||
'token' => $token,
|
||||
'tables' => ['users'],
|
||||
'format' => 'sql',
|
||||
]);
|
||||
|
||||
$convResponse->assertStatus(200);
|
||||
$this->assertEquals('application/sql', $convResponse->headers->get('Content-Type'));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user