From e447a2b3e4027c7281cbb52fa6e63cb1ce8ab8c9 Mon Sep 17 00:00:00 2001 From: Chetan baliyan Date: Sun, 13 Aug 2023 23:10:32 +0530 Subject: [PATCH] feat: add function to construct command to add multiple images --- src/images/imagesProccessing.ts | 123 ++++++++++++++++++++++++++++++++ src/index.ts | 8 +-- src/promptTemplates/image.ts | 41 +++++++++++ src/video/video.ts | 4 +- 4 files changed, 168 insertions(+), 8 deletions(-) create mode 100644 src/images/imagesProccessing.ts create mode 100644 src/promptTemplates/image.ts diff --git a/src/images/imagesProccessing.ts b/src/images/imagesProccessing.ts new file mode 100644 index 0000000..cc943fc --- /dev/null +++ b/src/images/imagesProccessing.ts @@ -0,0 +1,123 @@ +import { OpenAI } from 'langchain/llms/openai' +import { PromptTemplate } from 'langchain/prompts' +import ffmpeg from 'fluent-ffmpeg' +import { LLMChain } from 'langchain/chains' +import { imageTemp, tryy } from '../promptTemplates/image' +import path from 'ffmpeg-static' + +ffmpeg.setFfmpegPath(path!) + +import { Configuration, OpenAIApi } from 'openai' +import fs from 'fs' +import { exec } from 'child_process' + +const configuration = new Configuration({ + apiKey: process.env.OPENAI_API, +}) +const openai = new OpenAIApi(configuration) + +export const imageProccessing = async ({ language, topic }: { language?: string; topic?: string }) => { + const inputVideoPath = '/home/chetan/code/ts-content-gpt/tryyyyyyyyy.mp4' + const inputImagePath = '/home/chetan/code/ts-content-gpt/oo.jpg' + const outputVideoPath = '/home/chetan/code/ts-content-gpt/yyooooooo.mp4' + const targetTimestamp = '00:00:10.000' + + const queries = [ + { Query: 'Great Wall of China', timestamp: '1 -> 3' }, + { Query: 'longest wall in the world', timestamp: '5 -> 8' }, + // { Query: 'Ming Dynasty', timestamp: '7 -> 8' }, + // { Query: 'Chinese Empire', timestamp: '11 -> 12' }, + // { Query: 'brick, tamperth, and stone', timestamp: '17 -> 18' }, + ] + + interface IQuery { + Query: string + timestamp: string + } + let filter = '' + queries.forEach((query: IQuery, index) => { + const currIndex = index + 1 + const prevIndex = index + const totalQuery = queries.length + const lastIndex = index === totalQuery - 1 + + const [startingTime, endTime] = query.timestamp.split('->') + if (index === 0) { + filter += ` -filter_complex "[${prevIndex}:v][${currIndex}:v]overlay=25:25:enable='between(t,${startingTime.trim()},${endTime.trim()})'[v${currIndex}];[v${currIndex}]` + return + } + + if (lastIndex) { + filter += `[${currIndex}:v]overlay=25:25:enable='between(t,${startingTime.trim()},${endTime.trim()})'[v${currIndex}];[v${currIndex}]` + filter += `format=yuv420p[v]"` + return + } + + filter += `[${currIndex}:v]overlay=25:25:enable='between(t,${startingTime.trim()},${endTime.trim()})'[v${currIndex}];[v${currIndex}]` + }) + + // console.log('filter: ', filter) + + const mainFilter = `${path} -i ${inputVideoPath} -i ${inputImagePath} -i ${inputImagePath} ${filter} -map "[v]" -map 0:a -c:v libx264 -c:a copy ${outputVideoPath} ` + + console.log('mainFilter: ', mainFilter) + + const ll = `${path} -i ${inputVideoPath} -i ${inputVideoPath} -i ${inputImagePath} -filter_complex "[0:v][1:v]overlay=25:25:enable='between(t,1 , 2)'[v1];[v1][2:v]overlay=25:25:enable='between(t,3 , 4)'[v2];[v2]format=yuv420p[v]" -map "[v]" -map 0:a -c:v libx264 -c:a copy /home/chetan/code/ts-content-gpt/yyooooooo.mp4` + + // const command = `${path} -i ${inputVideoPath} -i ${inputImagePath} -filter_complex "[0:v][1:v] overlay=25:25:enable='between(t,0,10)'" -pix_fmt yuv420p -c:a copy ${outputVideoPath}` + // const command = `${path} -i ${inputVideoPath} -i ${inputImagePath} -filter_complex "[1:v]scale=-1:100[ovrl];[0:v][ovrl]overlay=x=W-w-10:y=10:enable='between(t,${targetTimestamp},T)'[out]" -map "[out]" -map 0:a -c:v libx264 -c:a copy ${outputVideoPath}` + // const i = `[0:v][1:v]overlay=25:25:enable='between(t,1 , 2)'[v1]; [v1][2:v]overlay=25:25:enable='between(t,3 , 4)'[v2]; [v2]format=yuv420p[v]" -map "[v]" -map 0:a -c:v libx264 -c:a copy /home/chetan/code/ts-content-gpt/yyooooooo.mp4` + // const command = `${path} -i ${inputVideoPath} -i ${inputImagePath} -i ${inputImagePath} -filter_complex "[0:v][1:v]overlay=25:25:enable='between(t,0,10)'[v1]; [v1][2:v]overlay=50:50:enable='between(t,15,20)'[v2]; [v2]format=yuv420p[v]" -map "[v]" -map 0:a -c:v libx264 -c:a copy ${outputVideoPath}` + // const command = `${path} -i ${inputVideoPath} -i ${inputImagePath} -i ${inputImagePath} -i ${inputImagePath} -filter_complex " [0:v][1:v]overlay=25:25:enable='between(t,0,10)'[v1];[v1] [2:v]overlay=50:50:enable='between(t,13,20)'[v2]; [v2][3:v]overlay=75:75:enable='between(t,23,30)'[v3]; [v3]format=yuv420p[v]" -map "[v]" -map 0:a -c:v libx264 -c:a copy ${outputVideoPath}` + + // const command = `${path} -i ${inputVideoPath} -i ${inputImagePath} -i ${inputImagePath} -i ${inputImagePath} -i ${inputImagePath} -i ${inputImagePath} -filter_complex "[0:v][1:v]overlay=25:25:enable='between(t,0,5)'[v1];[v1][2:v]overlay=50:50:enable='between(t,7,13)'[v2];[v2][3:v]overlay=75:75:enable='between(t,12,16)'[v3];[v3][4:v]overlay=100:100:enable='between(t,18,22)'[v4];[v4][5:v]overlay=125:125:enable='between(t,24,26)'[v5];[v5]format=yuv420p[v]" -map "[v]" -map 0:a -c:v libx264 -c:a copy ${outputVideoPath}` + + exec(mainFilter, (err, stdout, stderr) => { + if (err) { + console.error(err) + // console.log('stderr: ', stderr) + } + // console.log(stdout) + }) + + return + ffmpeg() + .input(inputVideoPath) + .input(inputImagePath) + .complexFilter([ + { + filter: 'setpts', + options: `PTS-STARTPTS+(gte(T,${targetTimestamp})*(${targetTimestamp}-PTS/TB))`, + outputs: 'timestamped', + }, + { + filter: 'overlay', + options: { + x: 'W-w-10', + y: '10', + }, + inputs: ['timestamped', '1'], // Use the 'timestamped' output from the previous filter and the image input + outputs: 'out', + }, + ]) + .outputOptions('-map', '[out]') + .outputOptions('-c:v', 'libx264') + .output(outputVideoPath) + .on('end', () => { + console.log('Image overlay complete!') + }) + .on('error', err => { + console.error(err) + }) + .run() + + return + fs.readFile('/home/chetan/code/ts-content-gpt/basicaudio.wav.srt', 'utf8', async function (err, data) { + if (err) throw err + const chatCompletion = await openai.createChatCompletion({ + model: 'gpt-3.5-turbo', + messages: [{ role: 'system', content: tryy(data) }], + }) + console.log('chatCompletion: ', chatCompletion.data.choices[0].message) + }) +} diff --git a/src/index.ts b/src/index.ts index a2dd856..ed25766 100644 --- a/src/index.ts +++ b/src/index.ts @@ -12,6 +12,7 @@ import { mergeAudio } from './video/video' import { uploadVideos } from './upoad/upload' import uploadFile from './upoad/azureUpload' +import { imageProccessing } from './images/imagesProccessing' const inputFilePath = path.join(__dirname, '..', 'basicaudio.mp3') const outputFilePath = path.join(__dirname, '..', 'basicaudio.wav') @@ -59,12 +60,7 @@ const generateYoutubeShort = async (language: string, topic: string) => { // app.use(express.json()) -mergeAudio({ - videoFilePath, - audioFilePath: outputFilePath, - outputVideoPath: outputVideoFilePath, -}) - +imageProccessing({ language: '' }) // app.post('/generate', async (req, res) => { // try { // const { language, topic } = req.body diff --git a/src/promptTemplates/image.ts b/src/promptTemplates/image.ts new file mode 100644 index 0000000..32a256f --- /dev/null +++ b/src/promptTemplates/image.ts @@ -0,0 +1,41 @@ +export const imageTemp = (timestamp: string) => ` +You are an expert video editor. you have to provide query to fetch images from an api . those images will be used by edior in video. The editor will give you the timestamp srt file of the video, + write querys for images to fetch.you have to analyse the timestamp and then give perfect image description , Make sure to directly write the query in response to the timestamp. you have to give perfect image description which we can show on perticular timestamp + make sure not to write 5 query (keep the query pretty short and neat). + You will produce purely text. + Don't write any other textual thing than the text itself. + + + +specifications delimited by angle brackets . + +TIMESTAMP: <${timestamp}> + + +# Output +You will output in parsable JSON object , all these json will be in a array. +[ + { + "query": "...", + "timestamp": "..." +}, +{ + "query": "...", + "timestamp": "..." +}, +//etc +] + +` + +export const tryy = ( + timestamp: string +) => `this is a timestamp for my video , i am using a api to fetch images , so based on this given timetamp you have to give me 5 best query by which i can fetch images also you have to send on what time i have to show that image , +Time is in formet of hh:mm:ss,ms --> hh:mm:ss,ms , so you only have to return seconds part of timestamp , for example if timestamp is 00:00:10,000 --> 00:00:15,000 , you only have to return 10 -> 15 , so i can show image between 10 to 15 seconds of video + +specifications delimited by angle brackets . +TIMESTAMP: <${timestamp}> + + +### OUTPUT +[{"Query":"...","timestamp":".. -> .."}]` diff --git a/src/video/video.ts b/src/video/video.ts index ee21a92..43df5bc 100644 --- a/src/video/video.ts +++ b/src/video/video.ts @@ -59,11 +59,11 @@ export const mergeAudio = async ({ const newSrtFilePath = path.join(__dirname, '..', '..', srtFilePath) const backgroundMusicFilePath = path.join(__dirname, '..', '..', 'bg.mp3') - const out = path.join(__dirname, '..', '..', 'tryyyyyyyyy.mp3') + const out = path.join(__dirname, '..', '..', 'tryyyyyyyyy.mp4') const subtitleStyle = "force_style='Alignment=6,FontName=Trebuchet,FontSize=18,PrimaryColour=&Hffffff&,OutlineColour=&H00000000&,MarginV=25'" - const backgroundAudiocommand = `${ffmpegPath} -i ${outputVideoPath} -i ${backgroundMusicFilePath} -filter_complex "[0:a]volume=1[a1];[1:a]volume=0.2[b1];[a1][b1]amix=inputs=2[aout]" -map 0:v -map "[aout]" -c:v copy -c:a aac -shortest ${out}` + const backgroundAudiocommand = `${ffmpegPath} -i ${outputVideoPath} -i ${backgroundMusicFilePath} -filter_complex "[0:a]volume=1[a1];[1:a]volume=0.4[b1];[a1][b1]amix=inputs=2[aout]" -map 0:v -map "[aout]" -c:v copy -c:a aac -shortest ${out}` return new Promise((resolve, reject) => { ffmpeg(videoFilePath)