From c9f94914a5b8d8255f471253fe74be20e0c66d1c Mon Sep 17 00:00:00 2001 From: Kevin Ngo Date: Tue, 9 Oct 2018 23:08:35 -0700 Subject: [PATCH] use audio buffer node to support slow down, pitch down, and audio effects --- src/components/beat-loader.js | 10 ++-- src/components/intro-song.js | 1 + src/components/song-preview.js | 1 + src/components/song.js | 103 ++++++++++++++++++++++----------- src/index.html | 4 +- src/state/index.js | 26 +++++++-- src/templates/stage.html | 2 +- 7 files changed, 101 insertions(+), 46 deletions(-) diff --git a/src/components/beat-loader.js b/src/components/beat-loader.js index b3c11ad..3ff0a1d 100644 --- a/src/components/beat-loader.js +++ b/src/components/beat-loader.js @@ -17,6 +17,7 @@ AFRAME.registerComponent('beat-loader', { verticalPositions: [1.00, 1.35, 1.70], init: function () { + this.audioAnalyserEl = document.getElementById('audioanalyser'); this.beams = document.getElementById('beams').components.beams; this.beatData = null; this.beatContainer = document.getElementById('beatContainer'); @@ -103,7 +104,6 @@ AFRAME.registerComponent('beat-loader', { * Generate beats and stuff according to timestamp. */ tick: function (time, delta) { - var audioEl = this.el.components.song.audio; var bpm; var i; var notes; @@ -112,11 +112,11 @@ AFRAME.registerComponent('beat-loader', { var msPerBeat; var noteTime; - if (!this.data.isPlaying || !this.data.challengeId || !this.beatData || !audioEl) { return; } + if (!this.data.isPlaying || !this.data.challengeId || !this.beatData) { return; } // Re-sync song with beats playback. - if (this.beatsTimeOffset !== undefined && this.songCurrentTime !== this.el.components.song.audio.currentTime) { - this.songCurrentTime = this.el.components.song.audio.currentTime; + if (this.beatsTimeOffset !== undefined && this.songCurrentTime !== this.el.components.song.context.currentTime) { + this.songCurrentTime = this.el.components.song.context.currentTime; this.beatsTime = (this.songCurrentTime + this.data.beatAnticipationTime) * 1000; } @@ -142,7 +142,7 @@ AFRAME.registerComponent('beat-loader', { if (this.beatsTimeOffset !== undefined) { if (this.beatsTimeOffset <= 0) { this.el.sceneEl.emit('beatloaderpreloadfinish', null, false); - this.songCurrentTime = this.el.components.song.audio.currentTime; + this.songCurrentTime = this.el.components.song.context.currentTime; this.beatsTimeOffset = undefined; } else { this.beatsTimeOffset -= delta; diff --git a/src/components/intro-song.js b/src/components/intro-song.js index a6e3f59..94973ad 100644 --- a/src/components/intro-song.js +++ b/src/components/intro-song.js @@ -15,6 +15,7 @@ AFRAME.registerComponent('intro-song', { if (!this.el.sceneEl.isPlaying) { return; } if (!oldData.isPlaying && this.data.isPlaying) { + this.analyserEl.components.audioanalyser.resumeContext(); this.timeout = setTimeout(() => { // TODO: Fade in volume. this.analyserEl.setAttribute('audioanalyser', 'src', audio); diff --git a/src/components/song-preview.js b/src/components/song-preview.js index 65ac6fe..ed99b2e 100644 --- a/src/components/song-preview.js +++ b/src/components/song-preview.js @@ -163,6 +163,7 @@ AFRAME.registerComponent('song-preview-system', { this.audio = this.audioStore[challengeId]; this.audio.volume = 0; this.volumeTarget.volume = 0; + this.analyserEl.components.audioanalyser.resumeContext(); this.audio.currentTime = this.audio.dataset.previewStartTime; this.audio.play(); this.animation.restart(); diff --git a/src/components/song.js b/src/components/song.js index 40aee6e..bc47398 100644 --- a/src/components/song.js +++ b/src/components/song.js @@ -1,5 +1,7 @@ const utils = require('../utils'); +var once = {once: true}; + /** * Active challenge song / audio. */ @@ -7,23 +9,28 @@ AFRAME.registerComponent('song', { schema: { analyserEl: {type: 'selector', default: '#audioAnalyser'}, challengeId: {default: ''}, + isBeatsPreloaded: {default: false}, + isGameOver: {default: false}, isPlaying: {default: false} }, init: function () { - // Use audio element for audioanalyser. - const audio = this.audio = document.createElement('audio'); - audio.setAttribute('id', 'song'); - audio.crossOrigin = 'anonymous'; - this.el.sceneEl.appendChild(audio); + this.analyserSetter = {buffer: true}; + this.context = this.data.analyserEl.components.audioanalyser.context; - this.el.addEventListener('pausemenuexit', () => { - audio.currentTime = 0; + // Restart, get new buffer source node and play. + this.el.addEventListener('pausemenurestart', () => { + this.source.disconnect(); + this.data.analyserEl.addEventListener('audioanalyserbuffersource', evt => { + this.source = evt.detail; + if (this.data.isBeatsPreloaded) { this.source.start(); } + }, once); + this.data.analyserEl.components.audioanalyser.refreshSource(); }); - this.el.addEventListener('pausemenurestart', () => { - audio.currentTime = 0; - audio.play(); + /* + this.el.addEventListener('pausemenuexit', () => { + this.data.analyserEl.components.audioanalyser.suspendContext(); }); audio.addEventListener('ended', () => { @@ -33,8 +40,7 @@ AFRAME.registerComponent('song', { audio.currentTime = 0; } }); - - this.el.addEventListener('slowdown', this.slowDown.bind(this)); + */ }, update: function (oldData) { @@ -42,33 +48,64 @@ AFRAME.registerComponent('song', { var el = this.el; var data = this.data; - // Changed challenge. - if (data.challengeId !== oldData.challengeId) { - let songUrl = utils.getS3FileUrl(data.challengeId, 'song.ogg'); - audio.setAttribute('src', data.challengeId ? songUrl : ''); + // Game over, slow down audio, and then stop. + if (!oldData.isGameOver && data.isGameOver) { + this.source.playbackRate = 0.5; + setTimeout(() => { + this.stopAudio(); + }, 1000); + return; } - // Keep playback state up to date. - if (data.isPlaying && data.challengeId && this.audio.paused) { - console.log(`Playing ${this.audio.src}...`); - this.data.analyserEl.setAttribute('audioanalyser', 'src', audio); - audio.playbackRate = 1; - audio.volume = 1; - audio.play(); + // New challenge, play if we have loaded and were waiting for beats to preload. + if (!oldData.isBeatsPreloaded && this.data.isBeatsPreloaded && this.source) { + this.source.start(); + } + + if (oldData.challengeId && !data.challengeId) { + this.stopAudio(); return; - } else if ((!data.isPlaying || !data.challengeId) && !audio.paused) { - audio.pause(); + } + + // New challenge, load audio and play when ready. + if (oldData.challengeId !== data.challengeId && data.challengeId) { + this.el.sceneEl.emit('songloadstart', null, false); + this.getAudio().then(source => { + this.el.sceneEl.emit('songloadfinish', null, false); + if (this.data.isBeatsPreloaded) { source.start(); } + }).catch(console.error); + } + + // Pause / stop. + if (oldData.isPlaying && !data.isPlaying) { + data.analyserEl.components.audioanalyser.suspendContext(); + } + + // Resume. + if (!oldData.isPlaying && data.isPlaying && this.source) { + data.analyserEl.components.audioanalyser.resumeContext(); } }, - slowDown: function (ev) { - var progress = ev.detail.progress; - if (progress > 0.01){ - this.audio.playbackRate = 0.5 + progress / 2.0; - this.audio.volume = progress; - } - else { - this.audio.pause(); + getAudio: function () { + const data = this.data; + return new Promise(resolve => { + data.analyserEl.addEventListener('audioanalyserbuffersource', evt => { + this.source = evt.detail; + resolve(this.source); + }, once); + this.analyserSetter.src = utils.getS3FileUrl(data.challengeId, 'song.ogg'); + data.analyserEl.setAttribute('audioanalyser', this.analyserSetter); + }); + }, + + stopAudio: function () { + if (!this.source) { + console.warn('[song] Tried to stopAudio, but not playing.'); + return; } + this.source.stop(); + this.source.disconnect(); + this.source = null; } }); diff --git a/src/index.html b/src/index.html index 3255a42..825a8f5 100644 --- a/src/index.html +++ b/src/index.html @@ -13,7 +13,7 @@ bind__beat-loader="challengeId: challenge.id; difficulty: challenge.difficulty; isPlaying: isPlaying" bind__gameover="isGameOver: isGameOver" bind__intro-song="isPlaying: menu.active && !menuSelectedChallenge.id" - bind__song="challengeId: challenge.id; isPlaying: isPlaying && !challenge.isLoading && !challenge.isPreloadingBeats" + bind__song="challengeId: challenge.id; isPlaying: isPlaying; isBeatsPreloaded: challenge.isBeatsPreloaded" bind__song-preview-system="selectedChallengeId: menuSelectedChallenge.id" bind__stage-colors="isGameOver: isGameOver" bind__overlay="enabled: !isPlaying" @@ -159,7 +159,7 @@ class="blade" geometry="primitive: box; height: 0.9; depth: 0.02; width: 0.02" material="shader: flat; color: {{ bladeColor }}" - play-sound="event: drawbladesound; sound: #saberDraw" + play-sound="event: drawbladesound; sound: #saberDraw; volume: 0.5" position="0 -0.55 0"> { - state.challenge.isPreloadingBeats = false; + state.challenge.isBeatsPreloaded = true; }, beatloaderstart: (state) => { - state.challenge.isPreloadingBeats = true; + state.challenge.isBeatsPreloaded = false; state.challenge.isLoading = true; }, @@ -142,6 +143,9 @@ AFRAME.registerState({ state.menuSelectedChallenge.image = utils.getS3FileUrl(id, 'image.jpg'); state.menuSelectedChallenge.downloadsText = `${challengeData.downloads} Plays`; computeMenuSelectedChallengeIndex(state); + + state.challenge.isLoading = true; + state.isSongLoading = true; }, menuchallengeunselect: () => { @@ -163,16 +167,19 @@ AFRAME.registerState({ pausemenurestart: (state) => { resetScore(state); + state.isBeatsPreloaded = false; state.isGameOver = false; state.isPaused = false; }, pausemenuexit: (state) => { resetScore(state); + state.isBeatsPreloaded = false; state.isGameOver = false; state.isPaused = false; state.isVictory = false; state.menu.active = true; + state.challenge.id = ''; }, /** @@ -224,6 +231,14 @@ AFRAME.registerState({ computeMenuSelectedChallengeIndex(state); }, + songloadfinish: (state) => { + state.isSongLoading = false; + }, + + songloadstart: (state) => { + state.isSongLoading = true; + }, + 'enter-vr': (state) => { state.inVR = true; }, @@ -241,7 +256,9 @@ AFRAME.registerState({ * Post-process the state after each action. */ computeState: (state) => { - state.isPlaying = !state.menu.active && !state.isPaused && !state.isVictory && !state.isGameOver; + state.isPlaying = + !state.menu.active && !state.isPaused && !state.isVictory && !state.isGameOver && + !state.challenge.isLoading && !state.isSongLoading; state.leftRaycasterActive = !state.isPlaying && state.activeHand === 'left' && state.inVR; state.rightRaycasterActive = !state.isPlaying && state.activeHand === 'right' && state.inVR; state.multiplierText = `${state.score.multiplier}x`; @@ -303,7 +320,6 @@ function checkGameOver (state) { if (state.damage >= DAMAGE_MAX) { state.damage = 0; state.isGameOver = true; - //state.isPaused = true; } } diff --git a/src/templates/stage.html b/src/templates/stage.html index f35fdf1..434f086 100644 --- a/src/templates/stage.html +++ b/src/templates/stage.html @@ -73,7 +73,7 @@ - +