How to make circular waveform video with background image using ffmpeg?

In this case, the geq filter needs the alpha parameter and the equation you wrote to calculate the transparency of pixels.

[v]format=rgba,geq='p(mod((2*W/(2*PI))*(PI+atan2(0.5*H-Y,X-W/2)),W), H-2*hypot(0.5*H-Y,X-W/2))':a="1*alpha(mod((2*W/(2*PI))*(PI+atan2(0.5*H-Y,X-W/2)),W), H-2*hypot(0.5*H-Y,X-W/2))"[vout]; // here

Full command:

ffmpeg -i input.mp3 -i background.png -filter_complex "[0:a]showwaves=size=100x100:colors=white:draw=full:mode=p2p[v];[v]format=rgba,
geq='p(mod((2*W/(2*PI))*(PI+atan2(0.5*H-Y,X-W/2)),W), H-2*hypot(0.5*H-Y,X-W/2))':a="1*alpha(mod((2*W/(2*PI))*(PI+atan2(0.5*H-Y,X-W/2)),W), H-2*hypot(0.5*H-Y,X-W/2))"[vout];[1:v][vout]overlay=(W-w)/2:(H-h)/2[outv]" -map "[outv]" -map 0:a -pix_fmt yuv420p output.mp4